{

constexpr size_t largeZeroFillThreshold = 1 * MB;

if (size >= largeZeroFillThreshold) {

size_t pageSizeValue = WTF::pageSize();

uintptr_t begin = reinterpret_cast<uintptr_t>(base);

uintptr_t end = begin + size;

uintptr_t pageAlignedBegin = roundUpToMultipleOf(pageSizeValue, begin);

uintptr_t pageAlignedEnd = roundDownToMultipleOf(pageSizeValue, end);

if (pageAlignedEnd > pageAlignedBegin) {

if (begin != pageAlignedBegin)

memset(base, 0, pageAlignedBegin - begin);

OSAllocator::zeroFill(reinterpret_cast<void*>(pageAlignedBegin), pageAlignedEnd - pageAlignedBegin);

if (end != pageAlignedEnd)

memset(reinterpret_cast<void*>(pageAlignedEnd), 0, end - pageAlignedEnd);

} else

memset(base, 0, size);

} else

memset(base, 0, size);

{

static LazyNeverDestroyed<Ref<SharedTask<void(void*)>>> destructor;

static std::once_flag onceKey;

std::call_once(onceKey, [&] {

destructor.construct(createSharedTask<void(void*)>([] (void* p) { Gigacage::free(Gigacage::Primitive, p); }));

});

return destructor.get().copyRef();

{

unsigned numTries = 2;

bool success = false;

for (unsigned i = 0; i < numTries && !success; ++i) {

switch (allocate()) {

case BufferMemoryResult::Kind::Success:

success = true;

break;

case BufferMemoryResult::Kind::SuccessAndNotifyMemoryPressure:

if (vm)

vm->heap.collectAsync(CollectionScope::Full);

success = true;

break;

case BufferMemoryResult::Kind::SyncTryToReclaimMemory:

if (i + 1 == numTries)

break;

if (vm)

vm->heap.collectSync(CollectionScope::Full);

break;

}

}

return success;

{

// Make sure malloc actually allocates something, but not too much. We use null to mean that the buffer is detached.

size_t initialBytes = roundUpToMultipleOf<PageCount::pageSize>(sizeInBytes);

if (!initialBytes)

initialBytes = PageCount::pageSize;

size_t maximumBytes = roundUpToMultipleOf<PageCount::pageSize>(maxByteLength);

if (!maximumBytes)

maximumBytes = PageCount::pageSize;

// The whole maximum is reserved up front while only the initial size is charged against the

// physical budget, so without this a single buffer could claim all the address space there is.

if (static_cast<uint64_t>(maximumBytes) > maxGrowableBufferReservationBytes)

return nullptr;

bool done = tryAllocate(vm,

[&] () -> BufferMemoryResult::Kind {

return BufferMemoryManager::singleton().tryAllocatePhysicalBytes(initialBytes);

});

if (!done)

return nullptr;

char* slowMemory = nullptr;

tryAllocate(vm,

[&] () -> BufferMemoryResult::Kind {

auto result = BufferMemoryManager::singleton().tryAllocateGrowableBoundsCheckingMemory(maximumBytes);

slowMemory = std::bit_cast<char*>(result.basePtr);

return result.kind;

});

if (!slowMemory) {

BufferMemoryManager::singleton().freePhysicalBytes(initialBytes);

return nullptr;

}

constexpr bool readable = false;

constexpr bool writable = false;

OSAllocator::protect(slowMemory + initialBytes, maximumBytes - initialBytes, readable, writable);

return adoptRef(*new BufferMemoryHandle(slowMemory, initialBytes, maximumBytes, PageCount::fromBytes(initialBytes), PageCount::fromBytes(maximumBytes), MemorySharingMode::Shared, MemoryMode::BoundsChecking));

: m_data(data)

, m_destructor(WTF::move(destructor))

, m_sizeInBytes(sizeInBytes)

, m_maxByteLength(maxByteLength.value_or(sizeInBytes))

, m_hasMaxByteLength(!!maxByteLength)

{

RELEASE_ASSERT(m_sizeInBytes <= MAX_ARRAY_BUFFER_SIZE);

: m_shared(WTF::move(shared))

, m_memoryHandle(m_shared->memoryHandle())

, m_sizeInBytes(m_shared->sizeInBytes(std::memory_order_seq_cst))

{

RELEASE_ASSERT(m_sizeInBytes <= MAX_ARRAY_BUFFER_SIZE);

bool adjustedForceFixedLengthIfWasm = forceFixedLengthIfWasm || !Options::useWasmMemoryToBufferAPIs();

if (m_shared->mode() == SharedArrayBufferContents::Mode::WebAssembly && adjustedForceFixedLengthIfWasm) {

m_hasMaxByteLength = false;

m_maxByteLength = m_sizeInBytes;

} else {

m_hasMaxByteLength = !!m_shared->maxByteLength();

m_maxByteLength = m_shared->maxByteLength().value_or(m_sizeInBytes);

}

// data() cannot destroy m_shared here so the code is safe as is so avoid

// refing for performance reasons.

SUPPRESS_UNCOUNTED_ARG m_data = DataType { m_shared->data() };

: m_data(data)

, m_memoryHandle(WTF::move(memoryHandle))

, m_sizeInBytes(sizeInBytes)

, m_maxByteLength(maxByteLength)

, m_hasMaxByteLength(true)

{

RELEASE_ASSERT(m_sizeInBytes <= MAX_ARRAY_BUFFER_SIZE);

{

CheckedSize sizeInBytes = numElements;

sizeInBytes *= elementByteSize;

if (sizeInBytes.hasOverflowed() || sizeInBytes.value() > MAX_ARRAY_BUFFER_SIZE) {

reset();

return;

}

size_t allocationSize = sizeInBytes.value();

if (!allocationSize)

allocationSize = 1; // Make sure malloc actually allocates something, but not too much. We use null to mean that the buffer is detached.

void* data = nullptr;

if (policy == InitializationPolicy::ZeroInitialize)

data = Gigacage::tryZeroedMalloc(Gigacage::Primitive, allocationSize);

else

data = Gigacage::tryMalloc(Gigacage::Primitive, allocationSize);

m_data = DataType(data);

if (!data) {

reset();

return;

}

m_sizeInBytes = sizeInBytes.value();

RELEASE_ASSERT(m_sizeInBytes <= MAX_ARRAY_BUFFER_SIZE);

m_maxByteLength = m_sizeInBytes;

m_hasMaxByteLength = false;

m_destructor = ArrayBuffer::primitiveGigacageDestructor();

{

m_shared = SharedArrayBufferContents::create(mutableSpan(), maxByteLength(), m_memoryHandle, WTF::move(m_destructor), SharedArrayBufferContents::Mode::Default);

m_destructor = nullptr;

{

ASSERT(!other.m_data);

other.tryAllocate(m_sizeInBytes, sizeof(char), ArrayBufferContents::InitializationPolicy::DontInitialize);

if (!other.m_data)

return;

memcpy(other.data(), data(), m_sizeInBytes);

other.m_sizeInBytes = m_sizeInBytes;

RELEASE_ASSERT(other.m_sizeInBytes <= MAX_ARRAY_BUFFER_SIZE);

ASSERT(other.m_maxByteLength <= MAX_ARRAY_BUFFER_SIZE);

{

ASSERT(!other.m_data);

ASSERT(m_shared);

other.m_data = m_data;

other.m_destructor = nullptr;

other.m_shared = m_shared;

other.m_memoryHandle = m_memoryHandle;

other.m_sizeInBytes = m_sizeInBytes;

other.m_maxByteLength = m_maxByteLength;

other.m_hasMaxByteLength = m_hasMaxByteLength;

RELEASE_ASSERT(other.m_sizeInBytes <= MAX_ARRAY_BUFFER_SIZE);

ASSERT(other.m_maxByteLength <= MAX_ARRAY_BUFFER_SIZE);

{

auto buffer = tryCreate(numElements, elementByteSize);

if (!buffer)

CRASH();

return buffer.releaseNonNull();

{

auto buffer = tryCreate(span);

if (!buffer)

CRASH();

return buffer.releaseNonNull();

{

ASSERT(!Gigacage::isEnabled() || (Gigacage::contains(data.data()) && Gigacage::contains(data.data() + data.size() - 1)));

return createFromBytes(data, ArrayBuffer::primitiveGigacageDestructor());

{

if (data.data() && !Gigacage::isCaged(Gigacage::Primitive, data.data()))

Gigacage::disablePrimitiveGigacage();

ArrayBufferContents contents(data, std::nullopt, WTF::move(destructor));

return create(WTF::move(contents));

{

ArrayBufferContents contents(WTF::move(shared), forceFixedLengthIfWasm);

return create(WTF::move(contents));

{

ArrayBufferContents contents;

contents.tryAllocate(span.size(), 1, ArrayBufferContents::InitializationPolicy::DontInitialize);

if (!contents.m_data)

return nullptr;

return createInternal(WTF::move(contents), span.data(), span.size());

{

auto buffer = tryCreate(numElements, elementByteSize, std::nullopt, policy);

if (!buffer)

CRASH();

return buffer.releaseNonNull();

{

auto buffer = adoptRef(*new ArrayBuffer(WTF::move(contents)));

if (byteLength) {

ASSERT(source);

memcpy(buffer->data(), source, byteLength);

}

return buffer;

{

if (!maxByteLength) {

ArrayBufferContents contents;

contents.tryAllocate(numElements, elementByteSize, policy);

if (!contents.m_data)

return nullptr;

return adoptRef(*new ArrayBuffer(WTF::move(contents)));

}

CheckedSize sizeInBytes = numElements;

sizeInBytes *= elementByteSize;

if (sizeInBytes.hasOverflowed() || sizeInBytes.value() > MAX_ARRAY_BUFFER_SIZE)

return nullptr;

if (sizeInBytes.value() > maxByteLength.value() || maxByteLength.value() > MAX_ARRAY_BUFFER_SIZE)

return nullptr;

auto handle = tryAllocateResizableMemory(nullptr, sizeInBytes.value(), maxByteLength.value());

if (!handle)

return nullptr;

void* memory = handle->memory();

ArrayBufferContents contents(memory, sizeInBytes.value(), maxByteLength.value(), handle.releaseNonNull());

return create(WTF::move(contents));

{

ASSERT(left <= right);

if (x < left)

x = left;

if (right < x)

x = right;

return x;

{

size_t currentLength = byteLength();

if (index < 0)

index = currentLength + index;

return clampValue(index, 0, currentLength);

{

size_t size = begin <= end ? end - begin : 0;

auto result = ArrayBuffer::tryCreate(span().subspan(begin, size));

if (result)

result->setSharingMode(sharingMode());

return result;

{

m_contents.makeShared();

pinAndLock();

ASSERT(!isDetached());

{

m_isWasmMemory = true;

pinAndLock();

ASSERT(!isDetachable());

{

ASSERT(isWasmMemory());

void* oldData = m_contents.data();

m_contents.refreshAfterWasmMemoryGrow(memory);

void* newData = m_contents.data();

if (newData == oldData)

return;

// JSArrayBufferViews (typed arrays) effectively cache their buffer's data pointer.

for (size_t i = numberOfIncomingReferences(); i--;) {

JSCell* cell = incomingReferenceAt(i);

auto* view = dynamicDowncast<JSArrayBufferView>(cell);

if (view)

view->refreshVector(newData);

}

{

if (newSharingMode == sharingMode())

return;

RELEASE_ASSERT(!isShared()); // Cannot revert sharing.

RELEASE_ASSERT(newSharingMode == ArrayBufferSharingMode::Shared);

makeShared();

{

if (!m_contents.m_data || !isShared()) {

result.m_data = nullptr;

return false;

}

m_contents.shareWith(result);

return true;

{

Ref<ArrayBuffer> protect(*this);

if (!m_contents.m_data) {

result.m_data = nullptr;

return false;

}

if (isShared()) {

m_contents.shareWith(result);

return true;

}

if (!isDetachable()) {

m_contents.copyTo(result);

if (!result.m_data)

return false;

return true;

}

result = m_contents.detach();

notifyDetaching(vm);

return true;

{

ASSERT(isWasmMemory());

auto unused = m_contents.detach();

notifyDetaching(vm);

{

for (size_t i = numberOfIncomingReferences(); i--;) {

JSCell* cell = incomingReferenceAt(i);

if (JSArrayBufferView* view = dynamicDowncast<JSArrayBufferView>(cell))

view->detachFromArrayBuffer();

}

m_detachingWatchpointSet.fireAll(vm, "Array buffer was detached");

{

auto shared = m_contents.m_shared;

if (!shared) [[unlikely]]

return makeUnexpected(GrowFailReason::GrowSharedUnavailable);

const bool requirePageMultiple = isWasmMemory();

auto result = shared->grow(vm, newByteLength, requirePageMultiple);

if (result && result.value() > 0)

vm.heap.reportExtraMemoryAllocated(static_cast<JSCell*>(nullptr), result.value());

return result;

{

RELEASE_ASSERT(!isWasmMemory());

auto memoryHandle = m_contents.m_memoryHandle;

if (!memoryHandle || m_contents.m_shared) [[unlikely]]

return makeUnexpected(GrowFailReason::GrowSharedUnavailable);

// A non-shared buffer that is not a Wasm memory owns its handle outright and is reachable only

// from its own agent's thread, so the handle's lock has nothing to guard here.

// Keep in mind that newByteLength may not be page-size-aligned.

if (m_contents.m_maxByteLength < newByteLength)

return makeUnexpected(GrowFailReason::InvalidGrowSize);

int64_t deltaByteLength = static_cast<int64_t>(newByteLength) - static_cast<int64_t>(m_contents.m_sizeInBytes);

if (!deltaByteLength)

return 0;

// A buffer's maxByteLength may exceed the region actually mapped for it, so growth is bounded by

// the mapping.

if (newByteLength > memoryHandle->mappedCapacity())

return makeUnexpected(GrowFailReason::WouldExceedMaximum);

auto newPageCount = PageCount::fromBytesWithRoundUp(newByteLength);

auto oldPageCount = PageCount::fromBytes(memoryHandle->size()); // MemoryHandle's size is always page-size aligned.

if (newPageCount.bytes() > MAX_ARRAY_BUFFER_SIZE)

return makeUnexpected(GrowFailReason::WouldExceedMaximum);

if (newPageCount != oldPageCount) {

ASSERT(memoryHandle->maximum() >= newPageCount);

size_t desiredSize = newPageCount.bytes();

RELEASE_ASSERT(desiredSize <= MAX_ARRAY_BUFFER_SIZE);

if (desiredSize > memoryHandle->size()) {

size_t bytesToAdd = desiredSize - memoryHandle->size();

ASSERT(bytesToAdd);

ASSERT(roundUpToMultipleOf<PageCount::pageSize>(bytesToAdd) == bytesToAdd);

bool allocationSuccess = tryAllocate(&vm,

[&] () -> BufferMemoryResult::Kind {

return BufferMemoryManager::singleton().tryAllocatePhysicalBytes(bytesToAdd);

});

if (!allocationSuccess)

return makeUnexpected(GrowFailReason::OutOfMemory);

void* memory = memoryHandle->memory();

RELEASE_ASSERT(memory);

// Signaling memory must have been pre-allocated virtually.

uint8_t* startAddress = static_cast<uint8_t*>(memory) + memoryHandle->size();

dataLogLnIf(ArrayBufferInternal::verbose, "Marking memory's ", RawPointer(memory), " as read+write in range [", RawPointer(startAddress), ", ", RawPointer(startAddress + bytesToAdd), ")");

constexpr bool readable = true;

constexpr bool writable = true;

OSAllocator::protect(startAddress, bytesToAdd, readable, writable);

} else {

size_t bytesToSubtract = memoryHandle->size() - desiredSize;

ASSERT(bytesToSubtract);

ASSERT(roundUpToMultipleOf<PageCount::pageSize>(bytesToSubtract) == bytesToSubtract);

BufferMemoryManager::singleton().freePhysicalBytes(bytesToSubtract);

void* memory = memoryHandle->memory();

RELEASE_ASSERT(memory);

// Signaling memory must have been pre-allocated virtually.

uint8_t* startAddress = static_cast<uint8_t*>(memory) + desiredSize;

dataLogLnIf(ArrayBufferInternal::verbose, "Marking memory's ", RawPointer(memory), " as none in range [", RawPointer(startAddress), ", ", RawPointer(startAddress + bytesToSubtract), ")");

constexpr bool readable = false;

constexpr bool writable = false;

OSAllocator::protect(startAddress, bytesToSubtract, readable, writable);

}

memoryHandle->updateSize(desiredSize);

}

if (m_contents.m_sizeInBytes < newByteLength)

zeroFill(std::bit_cast<uint8_t*>(data()) + m_contents.m_sizeInBytes, newByteLength - m_contents.m_sizeInBytes);

m_contents.m_sizeInBytes = newByteLength;

if (deltaByteLength > 0)

vm.heap.reportExtraMemoryAllocated(static_cast<JSCell*>(nullptr), deltaByteLength);

return deltaByteLength;

{

CheckedSize sizeInBytes = numElements;

sizeInBytes *= elementByteSize;

if (sizeInBytes.hasOverflowed() || sizeInBytes.value() > maxByteLength || maxByteLength > MAX_ARRAY_BUFFER_SIZE)

return nullptr;

auto handle = tryAllocateResizableMemory(&vm, sizeInBytes.value(), maxByteLength);

if (!handle)

return nullptr;

auto* memory = static_cast<uint8_t*>(handle->memory());

return createShared(SharedArrayBufferContents::create({ memory, sizeInBytes.value() }, maxByteLength, WTF::move(handle), nullptr, SharedArrayBufferContents::Mode::Default));

{

if (!m_hasMaxByteLength)

return makeUnexpected(GrowFailReason::GrowSharedUnavailable);

ASSERT(m_memoryHandle);

// Collections happen with the lock released, because asking for one can run finalizers on this

// thread; a retry therefore starts over, since another agent may have grown this buffer meanwhile.

constexpr unsigned maximumAttempts = 2;

for (unsigned attempt = 1; ; ++attempt) {

auto collection = BufferMemoryResult::Kind::Success;

Expected<int64_t, GrowFailReason> result;

{

Locker locker { m_memoryHandle->lock() };

result = tryGrow(locker, newByteLength, requirePageMultiple, collection);

}

switch (collection) {

case BufferMemoryResult::Kind::Success:

return result;

case BufferMemoryResult::Kind::SuccessAndNotifyMemoryPressure:

vm.heap.collectAsync(CollectionScope::Full);

return result;

case BufferMemoryResult::Kind::SyncTryToReclaimMemory:

if (attempt == maximumAttempts)

return makeUnexpected(GrowFailReason::OutOfMemory);

vm.heap.collectSync(CollectionScope::Full);

break;

}

}

{

// Keep in mind that newByteLength may not be page-size-aligned. If the buffer is a Wasm memory, that is an error.

size_t sizeInBytes = m_sizeInBytes.load(std::memory_order_seq_cst);

if (sizeInBytes > newByteLength || m_maxByteLength < newByteLength)

return makeUnexpected(GrowFailReason::InvalidGrowSize);

int64_t deltaByteLength = newByteLength - sizeInBytes;

#if ENABLE(WEBASSEMBLY)

if (Options::useWasmMemoryToBufferAPIs()) {

if (requirePageMultiple && deltaByteLength % PageCount::pageSize)

return makeUnexpected(GrowFailReason::InvalidGrowSize);

}

#else

UNUSED_PARAM(requirePageMultiple);

#endif

if (!deltaByteLength)

return 0;

// A buffer's maxByteLength may exceed the region actually mapped for it, so growth is bounded by

// the mapping.

if (newByteLength > m_memoryHandle->mappedCapacity())

return makeUnexpected(GrowFailReason::WouldExceedMaximum);

auto newPageCount = PageCount::fromBytesWithRoundUp(newByteLength);

auto oldPageCount = PageCount::fromBytes(m_memoryHandle->size()); // MemoryHandle's size is always page-size aligned.

if (newPageCount.bytes() > MAX_ARRAY_BUFFER_SIZE)

return makeUnexpected(GrowFailReason::WouldExceedMaximum);

RefPtr memoryHandle = m_memoryHandle;

if (newPageCount != oldPageCount) {

ASSERT(memoryHandle->maximum() >= newPageCount);

size_t desiredSize = newPageCount.bytes();

RELEASE_ASSERT(desiredSize <= MAX_ARRAY_BUFFER_SIZE);

RELEASE_ASSERT(desiredSize > memoryHandle->size());

size_t extraBytes = desiredSize - memoryHandle->size();

RELEASE_ASSERT(extraBytes);

collection = BufferMemoryManager::singleton().tryAllocatePhysicalBytes(extraBytes);

if (collection == BufferMemoryResult::Kind::SyncTryToReclaimMemory)

return makeUnexpected(GrowFailReason::OutOfMemory);

void* memory = memoryHandle->memory();

RELEASE_ASSERT(memory);

// Signaling memory must have been pre-allocated virtually.

uint8_t* startAddress = static_cast<uint8_t*>(memory) + memoryHandle->size();

dataLogLnIf(ArrayBufferInternal::verbose, "Marking memory's ", RawPointer(memory), " as read+write in range [", RawPointer(startAddress), ", ", RawPointer(startAddress + extraBytes), ")");

constexpr bool readable = true;

constexpr bool writable = true;

OSAllocator::protect(startAddress, extraBytes, readable, writable);

memoryHandle->updateSize(desiredSize);

}

zeroFill(std::bit_cast<uint8_t*>(data()) + sizeInBytes, newByteLength - sizeInBytes);

updateSize(newByteLength);

UNUSED_PARAM(locker);

#if ENABLE(WEBASSEMBLY)

// Update cache for instance

for (Ref anchor : memoryHandle->anchors(locker)) {

Locker locker { anchor->m_lock };

if (JSWebAssemblyInstance* instance = anchor->instance())

instance->updateCachedMemories();

}

#endif

return deltaByteLength;

{

ASSERT(!buffer->isDetachable());

if (buffer->isShared())

return "Cannot transfer a SharedArrayBuffer"_s;

if (buffer->isWasmMemory())

return "Cannot transfer a WebAssembly.Memory"_s;

return "Cannot transfer an ArrayBuffer whose backing store has been accessed by the JavaScriptCore C API"_s;

{

void* buffer = Gigacage::tryMalloc(Gigacage::Primitive, data.size_bytes());

if (!buffer)

return std::nullopt;

memcpy(buffer, data.data(), data.size_bytes());

return ArrayBufferContents { buffer, data.size_bytes(), std::nullopt, ArrayBuffer::primitiveGigacageDestructor() };

{

#if ENABLE(WEBASSEMBLY)

ASSERT(isResizableNonShared());

// If the memory is BoundChecking, the memory's handle is replaced with a different one when it grows.

m_memoryHandle = memory->handle();

m_data = memory->basePointer();

m_sizeInBytes = m_memoryHandle->size();

#else

UNUSED_PARAM(memory);

#endif