/*
* Copyright (C) 2012-2024 Apple Inc. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
* OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include "config.h"
#include "BlockDirectory.h"
#include "BlockDirectoryInlines.h"
#include "Heap.h"
#include "HeapInlines.h"
#include "MarkedSpaceInlines.h"
#include "SubspaceInlines.h"
#include "SuperSampler.h"
#include
#include
#include
namespace JSC {
namespace BlockDirectoryInternal {
static constexpr bool verbose = false;
}
DEFINE_ALLOCATOR_WITH_HEAP_IDENTIFIER(BlockDirectory);
BlockDirectory::BlockDirectory(size_t cellSize)
: m_cellSize(static_cast(cellSize))
{
}
BlockDirectory::~BlockDirectory()
{
Locker locker { m_localAllocatorsLock };
while (!m_localAllocators.isEmpty())
m_localAllocators.begin()->remove();
}
void BlockDirectory::setSubspace(Subspace* subspace)
{
m_attributes = subspace->attributes();
m_subspace = subspace;
}
void BlockDirectory::updatePercentageOfPagedOutPages(SimpleStats& stats)
{
// FIXME: We should figure out a solution for Windows and PlayStation.
// QNX doesn't have mincore(), though the information can be had. But since all mapped
// pages are resident, does it matter?
#if OS(UNIX) && !PLATFORM(PLAYSTATION) && !OS(QNX) && !OS(HAIKU)
size_t pageSize = WTF::pageSize();
ASSERT(!(MarkedBlock::blockSize % pageSize));
auto numberOfPagesInMarkedBlock = MarkedBlock::blockSize / pageSize;
// For some reason this can be unsigned char or char on different OSes...
using MincoreBufferType = std::remove_pointer_t;
static_assert(std::is_same_v);
Vector pagedBits(FillWith { }, numberOfPagesInMarkedBlock, MincoreBufferType { });
for (auto* handle : m_blocks) {
if (!handle)
continue;
auto* pageStart = handle->pageStart();
auto markedBlockSizeInBytes = handle->backingStorageSize();
RELEASE_ASSERT(markedBlockSizeInBytes / pageSize = m_blocks.size())
return nullptr;
dataLogLnIf(BlockDirectoryInternal::verbose, "Setting block ", unsweptCursor, " in use (findBlockToSweep) for ", *this);
setIsInUse(unsweptCursor, true);
return m_blocks[unsweptCursor];
}
void BlockDirectory::sweep()
{
// We need to be careful of a weird race where while we are sweeping a block
// the concurrent sweeper comes along and takes the inUse bit for a block
// in the same bit vector word as we're currently scanning. If we did't
// refresh our view into the word we could see stale data and try to scan
// a block already in use.
Locker locker(bitvectorLock());
for (size_t index = 0; index < m_blocks.size(); ++index) {
index = (unsweptBits() & ~inUseBits()).findBit(index, true);
if (index >= m_blocks.size())
break;
MarkedBlock::Handle* block = m_blocks[index];
ASSERT(!isInUse(index));
dataLogLnIf(BlockDirectoryInternal::verbose, "Setting block ", index, " in use (sweep) for ", *this);
setIsInUse(index, true);
{
DropLockForScope scope(locker);
block->sweep(nullptr);
}
ASSERT(!isUnswept(index));
setIsInUse(index, false);
}
}
void BlockDirectory::shrink()
{
// We need to be careful of a weird race where while we are sweeping a block
// the concurrent sweeper comes along and takes the inUse bit for a block
// in the same bit vector word as we're currently scanning. If we did't
// refresh our view into the word we could see stale data and try to scan
// a block already in use.
Locker locker(bitvectorLock());
for (size_t index = 0; index < m_blocks.size(); ++index) {
index = (emptyBits() & ~destructibleBits() & ~inUseBits()).findBit(index, true);
if (index >= m_blocks.size())
break;
ASSERT(!isInUse(index));
dataLogLnIf(BlockDirectoryInternal::verbose, "Setting block ", index, " in use (shrink) for ", *this);
setIsInUse(index, true);
{
DropLockForScope scope(locker);
markedSpace().freeBlock(m_blocks[index]);
}
setIsInUse(index, false);
}
}
// FIXME: rdar://139998916
MarkedBlock::Handle* BlockDirectory::findMarkedBlockHandleDebug(MarkedBlock* block)
{
for (size_t index = 0; index < m_blocks.size(); ++index) {
MarkedBlock::Handle* handle = m_blocks[index];
if (handle && &handle->block() == block)
return handle;
}
return nullptr;
}
void BlockDirectory::assertNoUnswept()
{
if (!ASSERT_ENABLED)
return;
assertIsMutatorOrMutatorIsStopped();
if (unsweptBitsView().isEmpty())
return;
dataLog("Assertion failed: unswept not empty in ", *this, ".\n");
dumpBits();
ASSERT_NOT_REACHED();
}
void BlockDirectory::didFinishUsingBlock(MarkedBlock::Handle* handle)
{
Locker locker(bitvectorLock());
didFinishUsingBlock(locker, handle);
}
void BlockDirectory::didFinishUsingBlock(AbstractLocker&, MarkedBlock::Handle* handle)
{
if (!isInUse(handle)) [[unlikely]] {
dataLogLn("Finish using on a block that's not in use: ", handle->index());
dumpBits();
RELEASE_ASSERT_NOT_REACHED();
}
dataLogLnIf(BlockDirectoryInternal::verbose, "Setting block ", handle->index(), " not in use (didFinishUsingBlock) for ", *this);
setIsInUse(handle, false);
}
RefPtr BlockDirectory::parallelNotEmptyBlockSource()
{
class Task final : public SharedTask {
public:
Task(BlockDirectory& directory)
: m_directory(directory)
{
}
MarkedBlock::Handle* run() final
{
if (m_done)
return nullptr;
Locker locker { m_lock };
m_directory.assertIsMutatorOrMutatorIsStopped();
m_index = m_directory.m_bits.markingNotEmpty().findBit(m_index, true);
if (m_index >= m_directory.m_blocks.size()) {
m_done = true;
return nullptr;
}
return m_directory.m_blocks[m_index++];
}
private:
BlockDirectory& m_directory WTF_GUARDED_BY_LOCK(m_lock);
size_t m_index WTF_GUARDED_BY_LOCK(m_lock) { 0 };
Lock m_lock;
bool m_done { false };
};
return adoptRef(new Task(*this));
}
void BlockDirectory::dump(PrintStream& out) const
{
out.print(RawPointer(this), ":", m_cellSize, "/", m_attributes);
}
void BlockDirectory::dumpBits(PrintStream& out)
{
unsigned maxNameLength = 0;
forEachBitVectorWithName(
[&](auto vectorRef, const char* name) {
UNUSED_PARAM(vectorRef);
WTF_ALLOW_UNSAFE_BUFFER_USAGE_BEGIN
unsigned length = strlen(name);
WTF_ALLOW_UNSAFE_BUFFER_USAGE_END
maxNameLength = std::max(maxNameLength, length);
});
forEachBitVectorWithName(
[&](auto vectorRef, const char* name) {
out.print(" ", name, ": ");
WTF_ALLOW_UNSAFE_BUFFER_USAGE_BEGIN
for (unsigned i = maxNameLength - strlen(name); i--;)
WTF_ALLOW_UNSAFE_BUFFER_USAGE_END
out.print(" ");
out.print(vectorRef, "\n");
});
}
MarkedSpace& BlockDirectory::markedSpace() const
{
return m_subspace->space();
}
#if ASSERT_ENABLED
void BlockDirectory::assertIsMutatorOrMutatorIsStopped() const
{
auto& heap = markedSpace().heap();
if (!heap.worldIsStopped()) {
if (auto owner = heap.vm().apiLock().ownerThread())
ASSERT(owner->get() == &Thread::currentSingleton());
else {
// FIXME: It feels like heap access should be tied to holding the API lock.
ASSERT(heap.hasAccess());
}
}
}
void BlockDirectory::assertSweeperIsSuspended() const
{
assertIsMutatorOrMutatorIsStopped();
}
#endif
} // namespace JSC