FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

inspector: add network payload buffer size limits · nodejs/node@9b83ef5 · GitHub

/ node Public

Commit 9b83ef5

Browse files
legendecas
authored and
aduh95
committed
inspector: add network payload buffer size limits
By default, the total buffered payloads are limited to 100MB, and 5MB for each single request. The oldest unused requests will be evicted first when the buffer size limit has been exceeded. PR-URL: #60236 Refs: https://chromedevtools.github.io/devtools-protocol/tot/Network/#method-enable Reviewed-By: Ryuhei Shima <shimaryuhei@gmail.com> Reviewed-By: Darshan Sen <raisinten@gmail.com>
1 parent 6ed6062 commit 9b83ef5

6 files changed

Lines changed: 434 additions & 53 deletions

File tree

‎node.gyp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -422,6 +422,7 @@
422422
'test/cctest/test_quic_tokens.cc',
423423
],
424424
'node_cctest_inspector_sources': [
425+
'test/cctest/inspector/test_network_requests_buffer.cc',
425426
'test/cctest/inspector/test_node_protocol.cc',
426427
'test/cctest/test_inspector_socket.cc',
427428
'test/cctest/test_inspector_socket_server.cc',

‎src/inspector/network_agent.cc‎

Lines changed: 42 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,8 @@ using v8::Object;
2525
using v8::Uint8Array;
2626
using v8::Value;
2727

28+
constexpr size_t kDefaultMaxTotalBufferSize = 100 * 1024 * 1024; // 100MB
29+
2830
// Get a protocol string property from the object.
2931
Maybe<protocol::String> ObjectGetProtocolString(v8::Local<v8::Context> context,
3032
Local<Object> object,
@@ -245,7 +247,8 @@ NetworkAgent::NetworkAgent(
245247
: inspector_(inspector),
246248
v8_inspector_(v8_inspector),
247249
env_(env),
248-
network_resource_manager_(std::move(network_resource_manager)) {
250+
network_resource_manager_(std::move(network_resource_manager)),
251+
requests_(kDefaultMaxTotalBufferSize) {
249252
event_notifier_map_["requestWillBeSent"] = &NetworkAgent::requestWillBeSent;
250253
event_notifier_map_["responseReceived"] = &NetworkAgent::responseReceived;
251254
event_notifier_map_["loadingFailed"] = &NetworkAgent::loadingFailed;
@@ -328,8 +331,15 @@ void NetworkAgent::Wire(protocol::UberDispatcher* dispatcher) {
328331
protocol::Network::Dispatcher::wire(dispatcher, this);
329332
}
330333

331-
protocol::DispatchResponse NetworkAgent::enable() {
334+
protocol::DispatchResponse NetworkAgent::enable(
335+
std::optional<int> in_maxTotalBufferSize,
336+
std::optional<int> in_maxResourceBufferSize) {
332337
inspector_->Enable();
338+
requests_ = RequestsBuffer(
339+
in_maxTotalBufferSize.value_or(kDefaultMaxTotalBufferSize));
340+
if (in_maxResourceBufferSize) {
341+
max_resource_buffer_size_ = *in_maxResourceBufferSize;
342+
}
333343
return protocol::DispatchResponse::Success();
334344
}
335345

@@ -340,7 +350,7 @@ protocol::DispatchResponse NetworkAgent::disable() {
340350

341351
protocol::DispatchResponse NetworkAgent::getRequestPostData(
342352
const protocol::String& in_requestId, protocol::String* out_postData) {
343-
auto request_entry = requests_.find(in_requestId);
353+
auto request_entry = requests_.cfind(in_requestId);
344354
if (request_entry == requests_.end()) {
345355
// Request not found, ignore it.
346356
return protocol::DispatchResponse::InvalidParams("Request not found");
@@ -361,7 +371,7 @@ protocol::DispatchResponse NetworkAgent::getRequestPostData(
361371

362372
// Concat response bodies.
363373
protocol::Binary buf =
364-
protocol::Binary::concat(request_entry->second.request_data_blobs);
374+
protocol::Binary::concat(request_entry->second.request_data_blobs());
365375
*out_postData = protocol::StringUtil::fromUTF8(buf.data(), buf.size());
366376
return protocol::DispatchResponse::Success();
367377
}
@@ -370,7 +380,7 @@ protocol::DispatchResponse NetworkAgent::getResponseBody(
370380
const protocol::String& in_requestId,
371381
protocol::String* out_body,
372382
bool* out_base64Encoded) {
373-
auto request_entry = requests_.find(in_requestId);
383+
auto request_entry = requests_.cfind(in_requestId);
374384
if (request_entry == requests_.end()) {
375385
// Request not found, ignore it.
376386
return protocol::DispatchResponse::InvalidParams("Request not found");
@@ -390,7 +400,7 @@ protocol::DispatchResponse NetworkAgent::getResponseBody(
390400

391401
// Concat response bodies.
392402
protocol::Binary buf =
393-
protocol::Binary::concat(request_entry->second.response_data_blobs);
403+
protocol::Binary::concat(request_entry->second.response_data_blobs());
394404
if (request_entry->second.response_charset == Charset::kBinary) {
395405
// If the response is binary, we return base64 encoded data.
396406
*out_body = buf.toBase64();
@@ -409,22 +419,26 @@ protocol::DispatchResponse NetworkAgent::getResponseBody(
409419

410420
protocol::DispatchResponse NetworkAgent::streamResourceContent(
411421
const protocol::String& in_requestId, protocol::Binary* out_bufferedData) {
412-
auto it = requests_.find(in_requestId);
413-
if (it == requests_.end()) {
414-
// Request not found, ignore it.
415-
return protocol::DispatchResponse::InvalidParams("Request not found");
416-
}
417-
auto& request_entry = it->second;
422+
bool is_response_finished = false;
423+
{
424+
auto it = requests_.find(in_requestId);
425+
if (it == requests_.end()) {
426+
// Request not found, ignore it.
427+
return protocol::DispatchResponse::InvalidParams("Request not found");
428+
}
429+
auto& request_entry = it->second;
418430

419-
request_entry.is_streaming = true;
431+
request_entry.is_streaming = true;
420432

421-
// Concat response bodies.
422-
*out_bufferedData =
423-
protocol::Binary::concat(request_entry.response_data_blobs);
424-
// Clear buffered data.
425-
request_entry.response_data_blobs.clear();
433+
// Concat response bodies.
434+
*out_bufferedData =
435+
protocol::Binary::concat(request_entry.response_data_blobs());
436+
// Clear buffered data.
437+
request_entry.clear_response_data_blobs();
438+
is_response_finished = request_entry.is_response_finished;
439+
}
426440

427-
if (request_entry.is_response_finished) {
441+
if (is_response_finished) {
428442
// If the request is finished, remove the entry.
429443
requests_.erase(in_requestId);
430444
}
@@ -499,9 +513,11 @@ void NetworkAgent::requestWillBeSent(v8::Local<v8::Context> context,
499513
}
500514

501515
auto request_charset = charset == "utf-8" ? Charset::kUTF8 : Charset::kBinary;
502-
requests_.emplace(
503-
request_id,
504-
RequestEntry(timestamp, request_charset, request->getHasPostData()));
516+
requests_.emplace(request_id,
517+
RequestEntry(timestamp,
518+
request_charset,
519+
request->getHasPostData(),
520+
max_resource_buffer_size_));
505521
frontend_->requestWillBeSent(request_id,
506522
std::move(request),
507523
std::move(initiator),
@@ -579,7 +595,7 @@ void NetworkAgent::loadingFinished(v8::Local<v8::Context> context,
579595

580596
frontend_->loadingFinished(request_id, timestamp);
581597

582-
auto request_entry = requests_.find(request_id);
598+
auto request_entry = requests_.cfind(request_id);
583599
if (request_entry == requests_.end()) {
584600
// No entry found. Ignore it.
585601
return;
@@ -589,7 +605,7 @@ void NetworkAgent::loadingFinished(v8::Local<v8::Context> context,
589605
// Streaming finished, remove the entry.
590606
requests_.erase(request_id);
591607
} else {
592-
request_entry->second.is_response_finished = true;
608+
requests_.find(request_id)->second.is_response_finished = true;
593609
}
594610
}
595611

@@ -630,7 +646,7 @@ void NetworkAgent::dataSent(v8::Local<v8::Context> context,
630646
}
631647
Local<Uint8Array> data = data_obj.As<Uint8Array>();
632648
auto data_bin = protocol::Binary::fromUint8Array(data);
633-
request_entry->second.request_data_blobs.push_back(data_bin);
649+
request_entry->second.push_request_data_blob(data_bin);
634650
}
635651

636652
void NetworkAgent::dataReceived(v8::Local<v8::Context> context,
@@ -672,7 +688,7 @@ void NetworkAgent::dataReceived(v8::Local<v8::Context> context,
672688
frontend_->dataReceived(
673689
request_id, timestamp, data_length, encoded_data_length, data_bin);
674690
} else {
675-
request_entry.response_data_blobs.push_back(data_bin);
691+
request_entry.push_response_data_blob(data_bin);
676692
}
677693
}
678694

‎src/inspector/network_agent.h‎

Lines changed: 10 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33

44
#include "env.h"
55
#include "io_agent.h"
6+
#include "network_requests_buffer.h"
67
#include "network_resource_manager.h"
78
#include "node/inspector/protocol/Network.h"
89

@@ -15,31 +16,6 @@ namespace inspector {
1516

1617
class NetworkInspector;
1718

18-
// Supported charsets for devtools frontend on request/response data.
19-
// If the charset is kUTF8, the data is expected to be text and can be
20-
// formatted on the frontend.
21-
enum class Charset {
22-
kUTF8,
23-
kBinary,
24-
};
25-
26-
struct RequestEntry {
27-
double timestamp;
28-
bool is_request_finished = false;
29-
bool is_response_finished = false;
30-
bool is_streaming = false;
31-
Charset request_charset;
32-
std::vector<protocol::Binary> request_data_blobs;
33-
Charset response_charset;
34-
std::vector<protocol::Binary> response_data_blobs;
35-
36-
RequestEntry(double timestamp, Charset request_charset, bool has_request_body)
37-
: timestamp(timestamp),
38-
is_request_finished(!has_request_body),
39-
request_charset(request_charset),
40-
response_charset(Charset::kBinary) {}
41-
};
42-
4319
class NetworkAgent : public protocol::Network::Backend {
4420
public:
4521
explicit NetworkAgent(
@@ -50,7 +26,9 @@ class NetworkAgent : public protocol::Network::Backend {
5026

5127
void Wire(protocol::UberDispatcher* dispatcher);
5228

53-
protocol::DispatchResponse enable() override;
29+
protocol::DispatchResponse enable(
30+
std::optional<int> in_maxTotalBufferSize,
31+
std::optional<int> in_maxResourceBufferSize) override;
5432

5533
protocol::DispatchResponse disable() override;
5634

@@ -104,12 +82,17 @@ class NetworkAgent : public protocol::Network::Backend {
10482
NetworkInspector* inspector_;
10583
v8_inspector::V8Inspector* v8_inspector_;
10684
std::shared_ptr<protocol::Network::Frontend> frontend_;
85+
10786
using EventNotifier = void (NetworkAgent::*)(v8::Local<v8::Context> context,
10887
v8::Local<v8::Object>);
10988
std::unordered_map<protocol::String, EventNotifier> event_notifier_map_;
110-
std::map<protocol::String, RequestEntry> requests_;
11189
Environment* env_;
11290
std::shared_ptr<NetworkResourceManager> network_resource_manager_;
91+
92+
// Per-resource buffer size in bytes to use when preserving network payloads
93+
// (XHRs, etc).
94+
size_t max_resource_buffer_size_ = 5 * 1024 * 1024; // 5MB
95+
RequestsBuffer requests_;
11396
};
11497

11598
} // namespace inspector

0 commit comments

Comments
 (0)
Back | FazBrowse Home | New Git URL