FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

Add measured peak memory to SpawnMetrics and execution log · codemod/bazel-java@b3126b5 · GitHub

  1. codemod
  2. bazel-java

Repository navigation

Commit b3126b5

Browse files
meisterT
authored and
copybara-github
committed
Add measured peak memory to SpawnMetrics and execution log
This change introduces measured peak memory usage (peak RSS) to the SpawnMetrics and exposes it in the execution log. Fixes bazelbuild#29872 Specifically: - Updates spawn.proto to add measured_memory_peak_bytes to SpawnMetrics. - Updates SpawnMetrics (Java) to store and propagate measuredMemoryPeakBytes. - Updates SpawnResult to populate this metric from the parsed execution statistics (memoryInKb). - Updates SpawnLogContext to write this metric to the execution log proto. - Updates linux-sandbox (C++) to read peak memory from cgroups (if available) and override the rusage maxrss value, providing high-accuracy aggregate memory usage for sandboxed actions on Linux. - Updates SpawnLogContextTestBase to verify the metric is correctly logged. PiperOrigin-RevId: 937171286 Change-Id: I4f563c7b8a361bb778f97e20cb3cd2ce9266f469
1 parent 080aa14 commit b3126b5

9 files changed

Lines changed: 170 additions & 2 deletions

File tree

‎src/main/java/com/google/devtools/build/lib/actions/SpawnMetrics.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ public class SpawnMetrics {
4545
this.inputBytes = builder.inputBytes;
4646
this.inputFiles = builder.inputFiles;
4747
this.memoryEstimateBytes = builder.memoryEstimateBytes;
48+
this.measuredMemoryPeakBytes = builder.measuredMemoryPeakBytes;
4849
}
4950

5051
/** Indicates whether the metrics correspond to the remote, local or worker execution. */
@@ -85,6 +86,7 @@ public String toString() {
8586
private final long inputBytes;
8687
private final long inputFiles;
8788
private final long memoryEstimateBytes;
89+
private final long measuredMemoryPeakBytes;
8890

8991
/** Any non-important stats < than 10% will not be shown in the summary. */
9092
private static final double STATS_SHOW_THRESHOLD = 0.10;
@@ -271,6 +273,15 @@ public long memoryEstimate() {
271273
return memoryEstimateBytes;
272274
}
273275

276+
/** Measured peak memory usage in bytes, or 0 if unavailable. */
277+
public long measuredMemoryPeak() {
278+
return measuredMemoryPeakBytes;
279+
}
280+
281+
public Builder toBuilder() {
282+
return Builder.forExec(execKind).addDurations(this).addNonDurations(this);
283+
}
284+
274285
/** Limit of total size in bytes of inputs or 0 if unavailable. */
275286
public long inputBytesLimit() {
276287
return 0;
@@ -317,6 +328,7 @@ public static class Builder {
317328
private long inputBytes = 0;
318329
private long inputFiles = 0;
319330
private long memoryEstimateBytes = 0;
331+
private long measuredMemoryPeakBytes = 0;
320332
long inputBytesLimit = 0;
321333
long inputFilesLimit = 0;
322334
long outputBytesLimit = 0;
@@ -524,6 +536,12 @@ public Builder setMemoryEstimateBytes(long memoryEstimateBytes) {
524536
return this;
525537
}
526538

539+
@CanIgnoreReturnValue
540+
public Builder setMeasuredMemoryPeakBytes(long measuredMemoryPeakBytes) {
541+
this.measuredMemoryPeakBytes = measuredMemoryPeakBytes;
542+
return this;
543+
}
544+
527545
@CanIgnoreReturnValue
528546
public Builder setInputBytesLimit(long inputBytesLimit) {
529547
this.inputBytesLimit = inputBytesLimit;
@@ -582,6 +600,7 @@ public Builder addNonDurations(SpawnMetrics metric) {
582600
inputFiles += metric.inputFiles();
583601
inputBytes += metric.inputBytes();
584602
memoryEstimateBytes += metric.memoryEstimate();
603+
measuredMemoryPeakBytes += metric.measuredMemoryPeak();
585604
inputFilesLimit += metric.inputFilesLimit();
586605
inputBytesLimit += metric.inputBytesLimit();
587606
outputFilesLimit += metric.outputFilesLimit();
@@ -596,6 +615,7 @@ public Builder maxNonDurations(SpawnMetrics metric) {
596615
inputFiles = Long.max(inputFiles, metric.inputFiles());
597616
inputBytes = Long.max(inputBytes, metric.inputBytes());
598617
memoryEstimateBytes = Long.max(memoryEstimateBytes, metric.memoryEstimate());
618+
measuredMemoryPeakBytes = Long.max(measuredMemoryPeakBytes, metric.measuredMemoryPeak());
599619
inputFilesLimit = Long.max(inputFilesLimit, metric.inputFilesLimit());
600620
inputBytesLimit = Long.max(inputBytesLimit, metric.inputBytesLimit());
601621
outputFilesLimit = Long.max(outputFilesLimit, metric.outputFilesLimit());

‎src/main/java/com/google/devtools/build/lib/actions/SpawnResult.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -316,10 +316,14 @@ final class SimpleSpawnResult implements SpawnResult {
316316
this.executorHostName = builder.executorHostName;
317317
this.runnerName = builder.runnerName;
318318
this.runnerSubtype = builder.runnerSubtype;
319-
this.spawnMetrics =
319+
SpawnMetrics metrics =
320320
builder.spawnMetrics != null
321321
? builder.spawnMetrics
322322
: SpawnMetrics.forLocalExecution(builder.wallTimeInMs);
323+
if (builder.memoryInKb != null) {
324+
metrics = metrics.toBuilder().setMeasuredMemoryPeakBytes(builder.memoryInKb * 1024).build();
325+
}
326+
this.spawnMetrics = metrics;
323327
this.startTime = builder.startTime;
324328
this.wallTimeInMs = builder.wallTimeInMs;
325329
this.userTimeInMs = builder.userTimeInMs;

‎src/main/java/com/google/devtools/build/lib/exec/SpawnLogContext.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -262,6 +262,7 @@ protected static Protos.SpawnMetrics getSpawnMetricsProto(SpawnResult result) {
262262
builder.setInputBytes(metrics.inputBytes());
263263
builder.setInputFiles(metrics.inputFiles());
264264
builder.setMemoryEstimateBytes(metrics.memoryEstimate());
265+
builder.setMeasuredMemoryPeakBytes(metrics.measuredMemoryPeak());
265266
builder.setInputBytesLimit(metrics.inputBytesLimit());
266267
builder.setInputFilesLimit(metrics.inputFilesLimit());
267268
builder.setOutputBytesLimit(metrics.outputBytesLimit());

‎src/main/protobuf/spawn.proto‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,8 @@ message SpawnMetrics {
114114
google.protobuf.Duration time_limit = 19;
115115
// Instant when the spawn started to execute.
116116
google.protobuf.Timestamp start_time = 20;
117+
// Measured peak memory usage in bytes, or 0 if unavailable.
118+
int64 measured_memory_peak_bytes = 21;
117119
}
118120

119121
// Details of an executed spawn.

‎src/main/tools/linux-sandbox.cc‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,10 @@
5858
#include <sys/wait.h>
5959
#include <unistd.h>
6060

61+
#include <algorithm>
6162
#include <atomic>
63+
#include <cinttypes>
64+
#include <cstdint>
6265
#include <string>
6366
#include <vector>
6467

@@ -214,6 +217,25 @@ static pid_t SpawnPid1() {
214217
return child_pid;
215218
}
216219

220+
static int64_t ReadCgroupsMemoryPeak(
221+
const std::vector<std::string>& cgroups_dirs) {
222+
int64_t max_value = -1;
223+
for (const std::string& dir : cgroups_dirs) {
224+
for (const char* fname : {"/memory.peak", "/memory.max_usage_in_bytes"}) {
225+
int64_t value = -1;
226+
std::string path = dir + fname;
227+
FILE* f = fopen(path.c_str(), "r");
228+
if (f != nullptr) {
229+
if (fscanf(f, "%" SCNd64, &value) == 1) {
230+
max_value = std::max(value, max_value);
231+
}
232+
fclose(f);
233+
}
234+
}
235+
}
236+
return max_value;
237+
}
238+
217239
static int WaitForPid1(const pid_t child_pid) {
218240
// Wait for the child to exit, obtaining usage information. Restart in the
219241
// case of a signal interrupting us.
@@ -239,6 +261,10 @@ static int WaitForPid1(const pid_t child_pid) {
239261

240262
// If we're supposed to write stats to a file, do so now.
241263
if (!opt.stats_path.empty()) {
264+
int64_t cgroups_memory_peak = ReadCgroupsMemoryPeak(opt.cgroups_dirs);
265+
if (cgroups_memory_peak > 0) {
266+
child_rusage.ru_maxrss = cgroups_memory_peak / 1024;
267+
}
242268
WriteStatsToFile(&child_rusage, opt.stats_path);
243269
}
244270

‎src/test/java/com/google/devtools/build/lib/actions/SpawnMetricsTest.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ public void builder_addDurationsNonDurations() throws Exception {
3232
.setInputBytes(10)
3333
.setInputFiles(20)
3434
.setMemoryEstimateBytes(30)
35+
.setMeasuredMemoryPeakBytes(35)
3536
.setInputBytesLimit(20)
3637
.setInputFilesLimit(40)
3738
.setOutputBytesLimit(50)
@@ -46,6 +47,7 @@ public void builder_addDurationsNonDurations() throws Exception {
4647
.setInputBytes(100)
4748
.setInputFiles(200)
4849
.setMemoryEstimateBytes(300)
50+
.setMeasuredMemoryPeakBytes(300)
4951
.setInputBytesLimit(200)
5052
.setInputFilesLimit(400)
5153
.setOutputBytesLimit(500)
@@ -67,6 +69,7 @@ public void builder_addDurationsNonDurations() throws Exception {
6769
assertThat(result.inputBytes()).isEqualTo(110);
6870
assertThat(result.inputFiles()).isEqualTo(220);
6971
assertThat(result.memoryEstimate()).isEqualTo(330);
72+
assertThat(result.measuredMemoryPeak()).isEqualTo(335);
7073
assertThat(result.inputBytesLimit()).isEqualTo(220);
7174
assertThat(result.inputFilesLimit()).isEqualTo(440);
7275
assertThat(result.outputBytesLimit()).isEqualTo(550);
@@ -84,6 +87,7 @@ public void builder_addDurationsMaxNonDurations() throws Exception {
8487
.setInputBytes(10)
8588
.setInputFiles(20)
8689
.setMemoryEstimateBytes(30)
90+
.setMeasuredMemoryPeakBytes(35)
8791
.setInputBytesLimit(20)
8892
.setInputFilesLimit(40)
8993
.setOutputBytesLimit(50)
@@ -98,6 +102,7 @@ public void builder_addDurationsMaxNonDurations() throws Exception {
98102
.setInputBytes(100)
99103
.setInputFiles(200)
100104
.setMemoryEstimateBytes(300)
105+
.setMeasuredMemoryPeakBytes(300)
101106
.setInputBytesLimit(200)
102107
.setInputFilesLimit(400)
103108
.setOutputBytesLimit(500)
@@ -119,6 +124,7 @@ public void builder_addDurationsMaxNonDurations() throws Exception {
119124
assertThat(result.inputBytes()).isEqualTo(100);
120125
assertThat(result.inputFiles()).isEqualTo(200);
121126
assertThat(result.memoryEstimate()).isEqualTo(300);
127+
assertThat(result.measuredMemoryPeak()).isEqualTo(300);
122128
assertThat(result.inputBytesLimit()).isEqualTo(200);
123129
assertThat(result.inputFilesLimit()).isEqualTo(400);
124130
assertThat(result.outputBytesLimit()).isEqualTo(500);

‎src/test/java/com/google/devtools/build/lib/actions/SpawnResultTest.java‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,4 +79,18 @@ public void inMemoryContents() {
7979
assertThat(r.getInMemoryOutput(null)).isEqualTo(null);
8080
assertThat(r.getInMemoryOutput(ActionInputHelper.fromPath("/does/not/exist"))).isEqualTo(null);
8181
}
82+
83+
@Test
84+
public void defaultSpawnMetricsIncludesMeasuredMemoryFromResourceUsage() {
85+
SpawnResult result =
86+
new SpawnResult.Builder()
87+
.setStatus(Status.SUCCESS)
88+
.setExitCode(0)
89+
.setRunnerName("test")
90+
.setWallTimeInMs(100)
91+
.setMemoryInKb(1000L)
92+
.build();
93+
94+
assertThat(result.getMetrics().measuredMemoryPeak()).isEqualTo(1000L * 1024L);
95+
}
8296
}

‎src/test/java/com/google/devtools/build/lib/exec/SpawnLogContextTestBase.java‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2074,7 +2074,11 @@ public void testTimeout() throws Exception {
20742074

20752075
@Test
20762076
public void testSpawnMetrics() throws Exception {
2077-
SpawnMetrics metrics = SpawnMetrics.Builder.forLocalExec().setTotalTimeInMs(1).build();
2077+
SpawnMetrics metrics =
2078+
SpawnMetrics.Builder.forLocalExec()
2079+
.setTotalTimeInMs(1)
2080+
.setMeasuredMemoryPeakBytes(42L)
2081+
.build();
20782082

20792083
SpawnLogContext context = createSpawnLogContext();
20802084

@@ -2093,6 +2097,7 @@ public void testSpawnMetrics() throws Exception {
20932097
.setMetrics(
20942098
Protos.SpawnMetrics.newBuilder()
20952099
.setTotalTime(millisToProto(1))
2100+
.setMeasuredMemoryPeakBytes(42L)
20962101
.setStartTime(Timestamps.fromDate(Date.from(now))))
20972102
.build());
20982103
}

‎src/test/shell/integration/linux-sandbox_test.sh‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -483,6 +483,96 @@ EOF
483483
expect_not_log "hi there"
484484
}
485485

486+
function test_cgroups_v2_peak_memory_mock() {
487+
local local_tmp="$(mktemp -d "${OUT_DIR}/test_cgroups_v2_peak_memory_mockXXXX")"
488+
local stats_out_path="${local_tmp}/statsfile"
489+
local stats_out_decoded_path="${local_tmp}/statsfile.decoded"
490+
491+
# Create a mock cgroup directory
492+
local mock_cgroup_dir="${local_tmp}/mock_cgroup"
493+
mkdir -p "${mock_cgroup_dir}"
494+
495+
# Write a mock peak memory value (40MB = 41943040 bytes)
496+
echo "41943040" > "${mock_cgroup_dir}/memory.peak"
497+
498+
# Run sandbox pointing to the mock cgroup directory
499+
local code=0
500+
"${linux_sandbox}" \
501+
-W "${SANDBOX_DIR}" \
502+
-S "${stats_out_path}" \
503+
-C "${mock_cgroup_dir}" \
504+
-- /bin/true \
505+
&> "${TEST_log}" || code="$?"
506+
507+
assert_equals 0 "${code}"
508+
509+
if ! [[ -e "${stats_out_path}" ]]; then
510+
fail "Stats file not found: '${stats_out_path}'"
511+
fi
512+
513+
# Decode stats
514+
"${protoc_compiler}" --proto_path="${STATS_PROTO_DIR}" \
515+
--decode tools.protos.ExecutionStatistics execution_statistics.proto \
516+
< "${stats_out_path}" > "${stats_out_decoded_path}"
517+
518+
if ! [[ -e "${stats_out_decoded_path}" ]]; then
519+
fail "Decoded stats file not found: '${stats_out_decoded_path}'"
520+
fi
521+
522+
# Assert maxrss is reported and matches the mock value (40MB = 40960 KB)
523+
local maxrss=0
524+
if grep -q maxrss "${stats_out_decoded_path}"; then
525+
maxrss="$(grep maxrss "${stats_out_decoded_path}" | cut -f2 -d':' | tr -dc '0-9')"
526+
fi
527+
528+
assert_equals 40960 "${maxrss}"
529+
}
530+
531+
function test_cgroups_v1_peak_memory_mock() {
532+
local local_tmp="$(mktemp -d "${OUT_DIR}/test_cgroups_v1_peak_memory_mockXXXX")"
533+
local stats_out_path="${local_tmp}/statsfile"
534+
local stats_out_decoded_path="${local_tmp}/statsfile.decoded"
535+
536+
# Create a mock cgroup directory
537+
local mock_cgroup_dir="${local_tmp}/mock_cgroup"
538+
mkdir -p "${mock_cgroup_dir}"
539+
540+
# Write a mock peak memory value (40MB = 41943040 bytes)
541+
echo "41943040" > "${mock_cgroup_dir}/memory.max_usage_in_bytes"
542+
543+
# Run sandbox pointing to the mock cgroup directory
544+
local code=0
545+
"${linux_sandbox}" \
546+
-W "${SANDBOX_DIR}" \
547+
-S "${stats_out_path}" \
548+
-C "${mock_cgroup_dir}" \
549+
-- /bin/true \
550+
&> "${TEST_log}" || code="$?"
551+
552+
assert_equals 0 "${code}"
553+
554+
if ! [[ -e "${stats_out_path}" ]]; then
555+
fail "Stats file not found: '${stats_out_path}'"
556+
fi
557+
558+
# Decode stats
559+
"${protoc_compiler}" --proto_path="${STATS_PROTO_DIR}" \
560+
--decode tools.protos.ExecutionStatistics execution_statistics.proto \
561+
< "${stats_out_path}" > "${stats_out_decoded_path}"
562+
563+
if ! [[ -e "${stats_out_decoded_path}" ]]; then
564+
fail "Decoded stats file not found: '${stats_out_decoded_path}'"
565+
fi
566+
567+
# Assert maxrss is reported and matches the mock value (40MB = 40960 KB)
568+
local maxrss=0
569+
if grep -q maxrss "${stats_out_decoded_path}"; then
570+
maxrss="$(grep maxrss "${stats_out_decoded_path}" | cut -f2 -d':' | tr -dc '0-9')"
571+
fi
572+
573+
assert_equals 40960 "${maxrss}"
574+
}
575+
486576
# The test shouldn't fail if the environment doesn't support running it.
487577
is_linux || exit 0
488578
check_sandbox_allowed || exit 0

0 commit comments

Comments
 (0)
Back | FazBrowse Home | New Git URL