1 /*
2 * Copyright (c) 2020, Red Hat Inc.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24 package jdk.test.lib.containers.cgroup;
25
26 import java.io.IOException;
27 import java.nio.file.Files;
28 import java.nio.file.Path;
29 import java.nio.file.Paths;
30 import java.util.Arrays;
31 import java.util.List;
32 import java.util.concurrent.TimeUnit;
33 import java.util.stream.Collectors;
34
35 import jdk.internal.platform.CgroupSubsystem;
36 import jdk.internal.platform.Metrics;
37
38 public class MetricsTesterCgroupV2 implements CgroupMetricsTester {
39
40 private static final long UNLIMITED = -1;
41 private static final UnifiedController UNIFIED = new UnifiedController();
42 private static final String MAX = "max";
43 private static final int PER_CPU_SHARES = 1024;
44
45 private final long startSysVal;
46 private final long startUserVal;
47 private final long startUsage;
48
49 static class UnifiedController {
50
51 private static final String NAME = "unified";
52 private final String path;
53
54 UnifiedController() {
55 path = constructPath();
56 }
57
58 String getPath() {
59 return path;
60 }
61
62 private static String constructPath() {
63 String mountPath;
64 String cgroupPath;
65 try {
66 List<String> fifthTokens = Files.lines(Paths.get("/proc/self/mountinfo"))
67 .filter( l -> l.contains("- cgroup2"))
68 .map(UnifiedController::splitAndMountPath)
69 .collect(Collectors.toList());
70 if (fifthTokens.size() != 1) {
71 throw new AssertionError("Expected only one cgroup2 line");
72 }
73 mountPath = fifthTokens.get(0);
74
75 List<String> cgroupPaths = Files.lines(Paths.get("/proc/self/cgroup"))
76 .filter( l -> l.startsWith("0:"))
77 .map(UnifiedController::splitAndCgroupPath)
78 .collect(Collectors.toList());
79 if (cgroupPaths.size() != 1) {
80 throw new AssertionError("Expected only one unified controller line");
81 }
82 cgroupPath = cgroupPaths.get(0);
83 return Paths.get(mountPath, cgroupPath).toString();
84 } catch (IOException e) {
85 return null;
86 }
87 }
88
89 public static String splitAndMountPath(String input) {
90 String[] tokens = input.split("\\s+");
91 return tokens[4]; // fifth entry is the mount path
92 }
93
94 public static String splitAndCgroupPath(String input) {
95 String[] tokens = input.split(":");
96 return tokens[2];
97 }
98 }
99
100 private long getLongLimitValueFromFile(String file) {
101 String strVal = getStringVal(file);
102 if (MAX.equals(strVal)) {
103 return UNLIMITED;
104 }
105 return convertStringToLong(strVal);
106 }
107
108 public MetricsTesterCgroupV2() {
109 Metrics metrics = Metrics.systemMetrics();
110 // Initialize CPU usage metrics before we do any testing.
111 startSysVal = metrics.getCpuSystemUsage();
112 startUserVal = metrics.getCpuUserUsage();
113 startUsage = metrics.getCpuUsage();
114 }
115
116 private long getLongValueFromFile(String file) {
117 return convertStringToLong(getStringVal(file));
118 }
119
120 private long getLongValueEntryFromFile(String file, String metric) {
121 Path filePath = Paths.get(UNIFIED.getPath(), file);
122 try {
123 String strVal = Files.lines(filePath).filter(l -> l.startsWith(metric)).collect(Collectors.joining());
124 String[] keyValues = strVal.split("\\s+");
125 String value = keyValues[1];
126 return convertStringToLong(value);
127 } catch (IOException e) {
128 return 0;
129 }
130 }
131
132 private String getStringVal(String file) {
133 Path filePath = Paths.get(UNIFIED.getPath(), file);
134 try {
135 return Files.lines(filePath).collect(Collectors.joining());
136 } catch (IOException e) {
137 return null;
138 }
139 }
140
141 private void fail(String metric, long oldVal, long newVal) {
142 CgroupMetricsTester.fail(UnifiedController.NAME, metric, oldVal, newVal);
143 }
144
145 private void fail(String metric, String oldVal, String newVal) {
146 CgroupMetricsTester.fail(UnifiedController.NAME, metric, oldVal, newVal);
147 }
148
149 private void warn(String metric, long oldVal, long newVal) {
150 CgroupMetricsTester.warn(UnifiedController.NAME, metric, oldVal, newVal);
151 }
152
153 private long getCpuShares(String file) {
154 long rawVal = getLongValueFromFile(file);
155 if (rawVal == 0 || rawVal == 100) {
156 return UNLIMITED;
157 }
158 int shares = (int)rawVal;
159 // CPU shares (OCI) value needs to get translated into
160 // a proper Cgroups v2 value. See:
161 // https://github.com/containers/crun/blob/master/crun.1.md#cpu-controller
162 //
163 // Use the inverse of (x == OCI value, y == cgroupsv2 value):
164 // ((262142 * y - 1)/9999) + 2 = x
165 //
166 int x = 262142 * shares - 1;
167 double frac = x/9999.0;
168 x = ((int)frac) + 2;
169 if ( x <= PER_CPU_SHARES ) {
170 return PER_CPU_SHARES; // mimic cgroups v1
171 }
172 int f = x/PER_CPU_SHARES;
173 int lower_multiple = f * PER_CPU_SHARES;
174 int upper_multiple = (f + 1) * PER_CPU_SHARES;
175 int distance_lower = Math.max(lower_multiple, x) - Math.min(lower_multiple, x);
176 int distance_upper = Math.max(upper_multiple, x) - Math.min(upper_multiple, x);
177 x = distance_lower <= distance_upper ? lower_multiple : upper_multiple;
178 return x;
179 }
180
181 private long getCpuMaxValueFromFile(String file) {
182 return getCpuValueFromFile(file, 0 /* $MAX index */);
183 }
184
185 private long getCpuPeriodValueFromFile(String file) {
186 return getCpuValueFromFile(file, 1 /* $PERIOD index */);
187 }
188
189 private long getCpuValueFromFile(String file, int index) {
190 String maxPeriod = getStringVal(file);
191 if (maxPeriod == null) {
192 return UNLIMITED;
193 }
194 String[] tokens = maxPeriod.split("\\s+");
195 String val = tokens[index];
196 if (MAX.equals(val)) {
197 return UNLIMITED;
198 }
199 return convertStringToLong(val);
200 }
201
202 private long convertStringToLong(String val) {
203 return CgroupMetricsTester.convertStringToLong(val, UNLIMITED);
204 }
205
206 @Override
207 public void testMemorySubsystem() {
208 Metrics metrics = Metrics.systemMetrics();
209
210 // User Memory
211 long oldVal = metrics.getMemoryFailCount();
212 long newVal = getLongValueEntryFromFile("memory.events", "max");
213 if (!CgroupMetricsTester.compareWithErrorMargin(oldVal, newVal)) {
214 fail("memory.events[max]", oldVal, newVal);
215 }
216
217 oldVal = metrics.getMemoryLimit();
218 newVal = getLongLimitValueFromFile("memory.max");
219 if (!CgroupMetricsTester.compareWithErrorMargin(oldVal, newVal)) {
220 fail("memory.max", oldVal, newVal);
221 }
222
223 oldVal = metrics.getMemoryUsage();
224 newVal = getLongValueFromFile("memory.current");
225 if (!CgroupMetricsTester.compareWithErrorMargin(oldVal, newVal)) {
226 fail("memory.current", oldVal, newVal);
227 }
228
229 oldVal = metrics.getTcpMemoryUsage();
230 newVal = getLongValueEntryFromFile("memory.stat", "sock");
231 if (!CgroupMetricsTester.compareWithErrorMargin(oldVal, newVal)) {
232 fail("memory.stat[sock]", oldVal, newVal);
233 }
234
235 oldVal = metrics.getMemoryAndSwapLimit();
236 newVal = getLongLimitValueFromFile("memory.swap.max");
237 if (!CgroupMetricsTester.compareWithErrorMargin(oldVal, newVal)) {
238 fail("memory.swap.max", oldVal, newVal);
239 }
240
241 oldVal = metrics.getMemoryAndSwapUsage();
242 newVal = getLongValueFromFile("memory.swap.current");
243 if (!CgroupMetricsTester.compareWithErrorMargin(oldVal, newVal)) {
244 fail("memory.swap.current", oldVal, newVal);
245 }
246
247 oldVal = metrics.getMemorySoftLimit();
248 newVal = getLongLimitValueFromFile("memory.high");
249 if (!CgroupMetricsTester.compareWithErrorMargin(oldVal, newVal)) {
250 fail("memory.high", oldVal, newVal);
251 }
252
253 }
254
255 @Override
256 public void testCpuAccounting() {
257 Metrics metrics = Metrics.systemMetrics();
258 long oldVal = metrics.getCpuUsage();
259 long newVal = TimeUnit.MICROSECONDS.toNanos(getLongValueEntryFromFile("cpu.stat", "usage_usec"));
260
261 if (!CgroupMetricsTester.compareWithErrorMargin(oldVal, newVal)) {
262 warn("cpu.stat[usage_usec]", oldVal, newVal);
263 }
264
265 oldVal = metrics.getCpuUserUsage();
266 newVal = TimeUnit.MICROSECONDS.toNanos(getLongValueEntryFromFile("cpu.stat", "user_usec"));
267 if (!CgroupMetricsTester.compareWithErrorMargin(oldVal, newVal)) {
268 warn("cpu.stat[user_usec]", oldVal, newVal);
269 }
270
271 oldVal = metrics.getCpuSystemUsage();
272 newVal = TimeUnit.MICROSECONDS.toNanos(getLongValueEntryFromFile("cpu.stat", "system_usec"));
273 if (!CgroupMetricsTester.compareWithErrorMargin(oldVal, newVal)) {
274 warn("cpu.stat[system_usec]", oldVal, newVal);
275 }
276 }
277
278 @Override
279 public void testCpuSchedulingMetrics() {
280 Metrics metrics = Metrics.systemMetrics();
281 long oldVal = metrics.getCpuPeriod();
282 long newVal = getCpuPeriodValueFromFile("cpu.max");
283 if (!CgroupMetricsTester.compareWithErrorMargin(oldVal, newVal)) {
284 fail("cpu.max[$PERIOD]", oldVal, newVal);
285 }
286
287 oldVal = metrics.getCpuQuota();
288 newVal = getCpuMaxValueFromFile("cpu.max");
289 if (!CgroupMetricsTester.compareWithErrorMargin(oldVal, newVal)) {
290 fail("cpu.max[$MAX]", oldVal, newVal);
291 }
292
293 oldVal = metrics.getCpuShares();
294 newVal = getCpuShares("cpu.weight");
295 if (!CgroupMetricsTester.compareWithErrorMargin(oldVal, newVal)) {
296 fail("cpu.weight", oldVal, newVal);
297 }
298
299 oldVal = metrics.getCpuNumPeriods();
300 newVal = getLongValueEntryFromFile("cpu.stat", "nr_periods");
301 if (!CgroupMetricsTester.compareWithErrorMargin(oldVal, newVal)) {
302 fail("cpu.stat[nr_periods]", oldVal, newVal);
303 }
304
305 oldVal = metrics.getCpuNumThrottled();
306 newVal = getLongValueEntryFromFile("cpu.stat", "nr_throttled");
307 if (!CgroupMetricsTester.compareWithErrorMargin(oldVal, newVal)) {
308 fail("cpu.stat[nr_throttled]", oldVal, newVal);
309 }
310
311 oldVal = metrics.getCpuThrottledTime();
312 newVal = TimeUnit.MICROSECONDS.toNanos(getLongValueEntryFromFile("cpu.stat", "throttled_usec"));
313 if (!CgroupMetricsTester.compareWithErrorMargin(oldVal, newVal)) {
314 fail("cpu.stat[throttled_usec]", oldVal, newVal);
315 }
316 }
317
318 @Override
319 public void testCpuSets() {
320 Metrics metrics = Metrics.systemMetrics();
321 int[] cpus = mapNullToEmpty(metrics.getCpuSetCpus());
322 Integer[] oldVal = Arrays.stream(cpus).boxed().toArray(Integer[]::new);
323 Arrays.sort(oldVal);
324
325 String cpusstr = getStringVal("cpuset.cpus");
326 // Parse range string in the format 1,2-6,7
327 Integer[] newVal = CgroupMetricsTester.convertCpuSetsToArray(cpusstr);
328 Arrays.sort(newVal);
329 if (Arrays.compare(oldVal, newVal) != 0) {
330 fail("cpuset.cpus", Arrays.toString(oldVal),
331 Arrays.toString(newVal));
332 }
333
334 cpus = mapNullToEmpty(metrics.getEffectiveCpuSetCpus());
335 oldVal = Arrays.stream(cpus).boxed().toArray(Integer[]::new);
336 Arrays.sort(oldVal);
337 cpusstr = getStringVal("cpuset.cpus.effective");
338 newVal = CgroupMetricsTester.convertCpuSetsToArray(cpusstr);
339 Arrays.sort(newVal);
340 if (Arrays.compare(oldVal, newVal) != 0) {
341 fail("cpuset.cpus.effective", Arrays.toString(oldVal),
342 Arrays.toString(newVal));
343 }
344
345 cpus = mapNullToEmpty(metrics.getCpuSetMems());
346 oldVal = Arrays.stream(cpus).boxed().toArray(Integer[]::new);
347 Arrays.sort(oldVal);
348 cpusstr = getStringVal("cpuset.mems");
349 newVal = CgroupMetricsTester.convertCpuSetsToArray(cpusstr);
350 Arrays.sort(newVal);
351 if (Arrays.compare(oldVal, newVal) != 0) {
352 fail("cpuset.mems", Arrays.toString(oldVal),
353 Arrays.toString(newVal));
354 }
355
356 cpus = mapNullToEmpty(metrics.getEffectiveCpuSetMems());
357 oldVal = Arrays.stream(cpus).boxed().toArray(Integer[]::new);
358 Arrays.sort(oldVal);
359 cpusstr = getStringVal("cpuset.mems.effective");
360 newVal = CgroupMetricsTester.convertCpuSetsToArray(cpusstr);
361 Arrays.sort(newVal);
362 if (Arrays.compare(oldVal, newVal) != 0) {
363 fail("cpuset.mems.effective", Arrays.toString(oldVal),
364 Arrays.toString(newVal));
365 }
366 }
367
368 private int[] mapNullToEmpty(int[] cpus) {
369 if (cpus == null) {
370 // Not available. For sake of testing continue with an
371 // empty array.
372 cpus = new int[0];
373 }
374 return cpus;
375 }
376
377 @Override
378 public void testCpuConsumption() {
379 Metrics metrics = Metrics.systemMetrics();
380 // make system call
381 long newSysVal = metrics.getCpuSystemUsage();
382 long newUserVal = metrics.getCpuUserUsage();
383 long newUsage = metrics.getCpuUsage();
384
385 // system/user CPU usage counters may be slowly increasing.
386 // allow for equal values for a pass
387 if (newSysVal < startSysVal) {
388 fail("getCpuSystemUsage", newSysVal, startSysVal);
389 }
390
391 // system/user CPU usage counters may be slowly increasing.
392 // allow for equal values for a pass
393 if (newUserVal < startUserVal) {
394 fail("getCpuUserUsage", newUserVal, startUserVal);
395 }
396
397 if (newUsage <= startUsage) {
398 fail("getCpuUsage", newUsage, startUsage);
399 }
400 }
401
402 @Override
403 public void testMemoryUsage() {
404 Metrics metrics = Metrics.systemMetrics();
405 long memoryUsage = metrics.getMemoryUsage();
406 long newMemoryUsage = 0;
407
408 // allocate memory in a loop and check more than once for new values
409 // otherwise we might occasionally see the effect of decreasing new memory
410 // values. For example because the system could free up memory
411 byte[][] bytes = new byte[32][];
412 for (int i = 0; i < 32; i++) {
413 bytes[i] = new byte[8*1024*1024];
414 newMemoryUsage = metrics.getMemoryUsage();
415 if (newMemoryUsage > memoryUsage) {
416 break;
417 }
418 }
419
420 if (newMemoryUsage < memoryUsage) {
421 fail("getMemoryUsage", memoryUsage, newMemoryUsage);
422 }
423 }
424
425 @Override
426 public void testMisc() {
427 testIOStat();
428 }
429
430 private void testIOStat() {
431 Metrics metrics = Metrics.systemMetrics();
432 long oldVal = metrics.getBlkIOServiceCount();
433 long newVal = getIoStatAccumulate(new String[] { "rios", "wios" });
434 if (!CgroupMetricsTester.compareWithErrorMargin(oldVal, newVal)) {
435 fail("io.stat->rios/wios: ", oldVal, newVal);
436 }
437
438 oldVal = metrics.getBlkIOServiced();
439 newVal = getIoStatAccumulate(new String[] { "rbytes", "wbytes" });
440 if (!CgroupMetricsTester.compareWithErrorMargin(oldVal, newVal)) {
441 fail("io.stat->rbytes/wbytes: ", oldVal, newVal);
442 }
443 }
444
445 private long getIoStatAccumulate(String[] matchNames) {
446 try {
447 return Files.lines(Paths.get(UNIFIED.getPath(), "io.stat"))
448 .map(line -> {
449 long accumulator = 0;
450 String[] tokens = line.split("\\s+");
451 for (String t: tokens) {
452 String[] keyVal = t.split("=");
453 if (keyVal.length != 2) {
454 continue;
455 }
456 for (String match: matchNames) {
457 if (match.equals(keyVal[0])) {
458 accumulator += Long.parseLong(keyVal[1]);
459 }
460 }
461 }
462 return accumulator;
463 }).collect(Collectors.summingLong(e -> e));
464 } catch (IOException e) {
465 return CgroupSubsystem.LONG_RETVAL_UNLIMITED;
466 }
467 }
468 }