/*
* Copyright (C) 2009-2023 Apple Inc. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
* OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#pragma once
#include
#if ENABLE(ASSEMBLER)
#define DUMP_LINK_STATISTICS 0
#define DUMP_CODE 0
#define GLOBAL_THUNK_ID reinterpret_cast(static_cast(-1))
#define CSS_CODE_ID reinterpret_cast(static_cast(-2))
#include
#include
#include
#include
#include
#include
#include
#include
#include
WTF_ALLOW_UNSAFE_BUFFER_USAGE_BEGIN
namespace JSC {
class SourceProvider;
class IRDumpDebugInfo {
WTF_MAKE_TZONE_ALLOCATED(IRDumpDebugInfo);
WTF_MAKE_NONCOPYABLE(IRDumpDebugInfo);
public:
// This is per-line either BasicBlock header annotation or actual IR node. Like,
// BB#0
// GetLocal
struct IRLine {
ASCIILiteral opName;
uint32_t blockIndex { 0 };
};
struct CodeEntry {
uint32_t codeOffset;
uint32_t irLineIndex;
};
IRDumpDebugInfo(CString&& name)
: functionName(WTF::move(name))
{
}
CString functionName;
Vector irLines;
Vector codeEntries;
};
class SourceCodeDumpDebugInfo {
WTF_MAKE_TZONE_ALLOCATED(SourceCodeDumpDebugInfo);
WTF_MAKE_NONCOPYABLE(SourceCodeDumpDebugInfo);
public:
struct CodeEntry {
uint32_t codeOffset;
LineColumn lineColumn;
Ref sourceProvider;
};
SourceCodeDumpDebugInfo(CString&& name)
: functionName(WTF::move(name))
{
}
CString functionName;
Vector codeEntries;
};
// LinkBuffer:
//
// This class assists in linking code generated by the macro assembler, once code generation
// has been completed, and the code has been copied to is final location in memory. At this
// time pointers to labels within the code may be resolved, and relative offsets to external
// addresses may be fixed.
//
// Specifically:
// * Jump objects may be linked to external targets,
// * The address of Jump objects may taken, such that it can later be relinked.
// * The return address of a Call may be acquired.
// * The address of a Label pointing into the code may be resolved.
// * The value referenced by a DataLabel may be set.
//
class LinkBuffer {
WTF_MAKE_NONCOPYABLE(LinkBuffer);
WTF_MAKE_TZONE_ALLOCATED(LinkBuffer);
template using CodeRef = MacroAssemblerCodeRef;
typedef MacroAssembler::Label Label;
typedef MacroAssembler::Jump Jump;
typedef MacroAssembler::PatchableJump PatchableJump;
typedef MacroAssembler::JumpList JumpList;
typedef MacroAssembler::Call Call;
typedef MacroAssembler::DataLabelCompact DataLabelCompact;
typedef MacroAssembler::DataLabel32 DataLabel32;
typedef MacroAssembler::DataLabelPtr DataLabelPtr;
typedef MacroAssembler::ConvertibleLoadLabel ConvertibleLoadLabel;
#if ENABLE(BRANCH_COMPACTION)
typedef MacroAssembler::LinkRecord LinkRecord;
typedef MacroAssembler::JumpLinkType JumpLinkType;
#endif
public:
#define FOR_EACH_LINKBUFFER_PROFILE(v) \
v(Baseline) \
v(DFG) \
v(FTL) \
v(DFGOSREntry) \
v(DFGOSRExit) \
v(FTLOSRExit) \
v(InlineCache) \
v(JumpIsland) \
v(Thunk) \
v(LLIntThunk) \
v(DFGThunk) \
v(FTLThunk) \
v(WasmThunk) \
v(ExtraCTIThunk) \
v(WasmOMG) \
v(WasmBBQ) \
v(YarrJIT) \
v(CSSJIT) \
v(Uncategorized) \
v(Total) \
#define DECLARE_LINKBUFFER_PROFILE(name) name,
enum class Profile {
FOR_EACH_LINKBUFFER_PROFILE(DECLARE_LINKBUFFER_PROFILE)
};
#undef DECLARE_LINKBUFFER_PROFILE
#define COUNT_LINKBUFFER_PROFILE(name) + 1
static constexpr unsigned numberOfProfiles = FOR_EACH_LINKBUFFER_PROFILE(COUNT_LINKBUFFER_PROFILE);
#undef COUNT_LINKBUFFER_PROFILE
static constexpr unsigned numberOfProfilesExcludingTotal = numberOfProfiles - 1;
enum class CacheFlushOnFinalize : bool { No, Yes };
LinkBuffer(MacroAssembler& macroAssembler, void* ownerUID, Profile profile = Profile::Uncategorized, JITCompilationEffort effort = JITCompilationMustSucceed, CacheFlushOnFinalize cacheFlushOnFinalize = CacheFlushOnFinalize::Yes)
: m_ownerUID(ownerUID)
, m_cacheFlushOnFinalize(cacheFlushOnFinalize)
, m_profile(profile)
{
linkCode(macroAssembler, effort);
}
template
LinkBuffer(MacroAssembler& macroAssembler, CodePtr code, size_t size, Profile profile = Profile::Uncategorized, JITCompilationEffort effort = JITCompilationMustSucceed, bool shouldPerformBranchCompaction = true)
: m_size(size)
, m_isRewriting(true)
, m_profile(profile)
, m_code(code.template retagged())
{
#if ENABLE(BRANCH_COMPACTION)
m_shouldPerformBranchCompaction = shouldPerformBranchCompaction;
#else
UNUSED_PARAM(shouldPerformBranchCompaction);
#endif
linkCode(macroAssembler, effort);
}
~LinkBuffer()
{
}
bool didFailToAllocate() const
{
return !m_didAllocate;
}
bool isValid() const
{
return !didFailToAllocate();
}
void setIsJumpIsland()
{
#if ASSERT_ENABLED
m_isJumpIsland = true;
#endif
}
// These methods are used to link or set values at code generation time.
template
requires (std::is_function_v)
void link(Call call, Func funcName)
{
CodePtr function(funcName);
link(call, function);
}
template
void link(Call call, CodePtr function)
{
ASSERT(call.isFlagSet(Call::Linkable));
call.m_label = applyOffset(call.m_label);
MacroAssembler::linkCall(code(), call, function);
}
template
void link(Call call, CodeLocationLabel label)
{
link(call, CodePtr(label));
}
template
void link(Jump jump, CodeLocationLabel label)
{
jump.m_label = applyOffset(jump.m_label);
MacroAssembler::linkJump(code(), jump, label);
}
template
void link(const JumpList& list, CodeLocationLabel label)
{
for (const Jump& jump : list.jumps())
link(jump, label);
}
void patch(DataLabelPtr label, void* value)
{
AssemblerLabel target = applyOffset(label.m_label);
MacroAssembler::linkPointer(code(), target, value);
}
template
void patch(DataLabelPtr label, CodeLocationLabel value)
{
AssemblerLabel target = applyOffset(label.m_label);
MacroAssembler::linkPointer(code(), target, value);
}
// These methods are used to obtain handles to allow the code to be relinked / repatched later.
template
CodeLocationLabel entrypoint() const
{
return CodeLocationLabel(tagCodePtr(code()));
}
template
CodeLocationCall locationOf(Call call)
{
ASSERT(call.isFlagSet(Call::Linkable));
ASSERT(!call.isFlagSet(Call::Near));
return CodeLocationCall(getLinkerAddress(applyOffset(call.m_label)));
}
template
CodeLocationNearCall locationOfNearCall(Call call)
{
ASSERT(call.isFlagSet(Call::Linkable));
ASSERT(call.isFlagSet(Call::Near));
return CodeLocationNearCall(getLinkerAddress(applyOffset(call.m_label)),
call.isFlagSet(Call::Tail) ? NearCallMode::Tail : NearCallMode::Regular);
}
template
CodeLocationLabel locationOf(PatchableJump jump)
{
return CodeLocationLabel(getLinkerAddress(applyOffset(jump.m_jump.m_label)));
}
template
CodeLocationLabel locationOf(Label label)
{
return CodeLocationLabel(getLinkerAddress(applyOffset(label.m_label)));
}
template
CodeLocationDataLabelPtr locationOf(DataLabelPtr label)
{
return CodeLocationDataLabelPtr(getLinkerAddress(applyOffset(label.m_label)));
}
template
CodeLocationDataLabel32 locationOf(DataLabel32 label)
{
return CodeLocationDataLabel32(getLinkerAddress(applyOffset(label.m_label)));
}
template
CodeLocationDataLabelCompact locationOf(DataLabelCompact label)
{
return CodeLocationDataLabelCompact(getLinkerAddress(applyOffset(label.m_label)));
}
template
CodeLocationConvertibleLoad locationOf(ConvertibleLoadLabel label)
{
return CodeLocationConvertibleLoad(getLinkerAddress(applyOffset(label.m_label)));
}
// This method obtains the return address of the call, given as an offset from
// the start of the code.
unsigned returnAddressOffset(Call call)
{
call.m_label = applyOffset(call.m_label);
return MacroAssembler::getLinkerCallReturnOffset(call);
}
uint32_t offsetOf(Label label)
{
return applyOffset(label.m_label).offset();
}
unsigned offsetOf(PatchableJump jump)
{
return applyOffset(jump.m_jump.m_label).offset();
}
// Upon completion of all patching 'FINALIZE_CODE()' should be called once to
// complete generation of the code. Alternatively, call
// finalizeCodeWithoutDisassembly() directly if you have your own way of
// displaying disassembly.
template
CodeRef finalizeCodeWithoutDisassembly(ASCIILiteral simpleName)
{
return finalizeCodeWithoutDisassemblyImpl(simpleName).template retagged();
}
template
CodeRef finalizeCodeWithDisassembly(bool dumpDisassembly, ASCIILiteral simpleName, const char* format, Args... args)
{
ALLOW_NONLITERAL_FORMAT_BEGIN
IGNORE_WARNINGS_BEGIN("format-security")
return finalizeCodeWithDisassemblyImpl(dumpDisassembly, simpleName, format, args...).template retagged();
IGNORE_WARNINGS_END
ALLOW_NONLITERAL_FORMAT_END
}
template
CodePtr trampolineAt(Label label)
{
return CodePtr(MacroAssembler::AssemblerType_T::getRelocatedAddress(code(), applyOffset(label.m_label)));
}
void* debugAddress()
{
return m_code.dataLocation();
}
size_t NODELETE size() const { return m_size; }
bool wasAlreadyDisassembled() const { return m_alreadyDisassembled; }
void didAlreadyDisassemble() { m_alreadyDisassembled = true; }
JS_EXPORT_PRIVATE static void NODELETE clearProfileStatistics();
JS_EXPORT_PRIVATE static void dumpProfileStatistics(std::optional = std::nullopt);
void setIsThunk() { m_isThunk = true; }
void setIRDumpDebugInfo(std::unique_ptr&& info) { m_irDumpDebugInfo = WTF::move(info); }
void setSourceCodeDumpDebugInfo(std::unique_ptr&& info) { m_sourceCodeDebugInfo = WTF::move(info); }
private:
JS_EXPORT_PRIVATE CodeRef finalizeCodeWithoutDisassemblyImpl(ASCIILiteral);
JS_EXPORT_PRIVATE CodeRef finalizeCodeWithDisassemblyImpl(bool dumpDisassembly, ASCIILiteral, const char* format, ...) WTF_ATTRIBUTE_PRINTF(4, 5);
#if ENABLE(BRANCH_COMPACTION)
int executableOffsetFor(int location)
{
// Returning 0 in this case works because at location <
// sizeof(int32_t), no compaction could have happened before this
// point as the assembler could not have placed a branch instruction
// within this space that required compaction.
if (location < static_cast(sizeof(int32_t)))
return 0;
return std::bit_cast(m_assemblerStorage.buffer())[location / sizeof(int32_t) - 1];
}
#endif
template T applyOffset(T src)
{
#if ENABLE(BRANCH_COMPACTION)
src = src.labelAtOffset(-executableOffsetFor(src.offset()));
#endif
return src;
}
// Keep this private! - the underlying code should only be obtained externally via finalizeCode().
void* code() const
{
return m_code.dataLocation();
}
void linkComments(MacroAssembler&);
void allocate(MacroAssembler&, JITCompilationEffort);
template
void* getLinkerAddress(T src)
{
void *code = this->code();
void* address = MacroAssembler::getLinkerAddress(code, src);
RELEASE_ASSERT(code