WTF_MAKE_TZONE_ALLOCATED(IRDumpDebugInfo);

WTF_MAKE_NONCOPYABLE(IRDumpDebugInfo);

// This is per-line either BasicBlock header annotation or actual IR node. Like,

// BB#0

// GetLocal

struct IRLine {

ASCIILiteral opName;

uint32_t blockIndex { 0 };

};

struct CodeEntry {

uint32_t codeOffset;

uint32_t irLineIndex;

};

IRDumpDebugInfo(CString&& name)

: functionName(WTF::move(name))

{

}

CString functionName;

Vector<IRLine> irLines;

Vector<CodeEntry> codeEntries;

WTF_MAKE_TZONE_ALLOCATED(SourceCodeDumpDebugInfo);

WTF_MAKE_NONCOPYABLE(SourceCodeDumpDebugInfo);

struct CodeEntry {

uint32_t codeOffset;

LineColumn lineColumn;

Ref<SourceProvider> sourceProvider;

};

SourceCodeDumpDebugInfo(CString&& name)

: functionName(WTF::move(name))

{

}

CString functionName;

Vector<CodeEntry> codeEntries;

WTF_MAKE_NONCOPYABLE(LinkBuffer);

WTF_MAKE_TZONE_ALLOCATED(LinkBuffer);

template<PtrTag tag> using CodeRef = MacroAssemblerCodeRef<tag>;

typedef MacroAssembler::Label Label;

typedef MacroAssembler::Jump Jump;

typedef MacroAssembler::PatchableJump PatchableJump;

typedef MacroAssembler::JumpList JumpList;

typedef MacroAssembler::Call Call;

typedef MacroAssembler::DataLabelCompact DataLabelCompact;

typedef MacroAssembler::DataLabel32 DataLabel32;

typedef MacroAssembler::DataLabelPtr DataLabelPtr;

typedef MacroAssembler::ConvertibleLoadLabel ConvertibleLoadLabel;

#if ENABLE(BRANCH_COMPACTION)

typedef MacroAssembler::LinkRecord LinkRecord;

typedef MacroAssembler::JumpLinkType JumpLinkType;

#endif

#define FOR_EACH_LINKBUFFER_PROFILE(v) \

v(Baseline) \

v(DFG) \

v(FTL) \

v(DFGOSREntry) \

v(DFGOSRExit) \

v(FTLOSRExit) \

v(InlineCache) \

v(JumpIsland) \

v(Thunk) \

v(LLIntThunk) \

v(DFGThunk) \

v(FTLThunk) \

v(WasmThunk) \

v(ExtraCTIThunk) \

v(WasmOMG) \

v(WasmBBQ) \

v(YarrJIT) \

v(CSSJIT) \

v(Uncategorized) \

v(Total) \

#define DECLARE_LINKBUFFER_PROFILE(name) name,

enum class Profile {

FOR_EACH_LINKBUFFER_PROFILE(DECLARE_LINKBUFFER_PROFILE)

};

#undef DECLARE_LINKBUFFER_PROFILE

#define COUNT_LINKBUFFER_PROFILE(name) + 1

static constexpr unsigned numberOfProfiles = FOR_EACH_LINKBUFFER_PROFILE(COUNT_LINKBUFFER_PROFILE);

#undef COUNT_LINKBUFFER_PROFILE

static constexpr unsigned numberOfProfilesExcludingTotal = numberOfProfiles - 1;

enum class CacheFlushOnFinalize : bool { No, Yes };

LinkBuffer(MacroAssembler& macroAssembler, void* ownerUID, Profile profile = Profile::Uncategorized, JITCompilationEffort effort = JITCompilationMustSucceed, CacheFlushOnFinalize cacheFlushOnFinalize = CacheFlushOnFinalize::Yes)

: m_ownerUID(ownerUID)

, m_cacheFlushOnFinalize(cacheFlushOnFinalize)

, m_profile(profile)

{

linkCode(macroAssembler, effort);

}

template<PtrTag tag>

LinkBuffer(MacroAssembler& macroAssembler, CodePtr<tag> code, size_t size, Profile profile = Profile::Uncategorized, JITCompilationEffort effort = JITCompilationMustSucceed, bool shouldPerformBranchCompaction = true)

: m_size(size)

, m_isRewriting(true)

, m_profile(profile)

, m_code(code.template retagged<LinkBufferPtrTag>())

{

#if ENABLE(BRANCH_COMPACTION)

m_shouldPerformBranchCompaction = shouldPerformBranchCompaction;

#else

UNUSED_PARAM(shouldPerformBranchCompaction);

#endif

linkCode(macroAssembler, effort);

}

~LinkBuffer()

{

}

bool didFailToAllocate() const

{

return !m_didAllocate;

}

bool isValid() const

{

return !didFailToAllocate();

}

void setIsJumpIsland()

{

#if ASSERT_ENABLED

m_isJumpIsland = true;

#endif

}

// These methods are used to link or set values at code generation time.

template<PtrTag tag, typename Func>

requires (std::is_function_v<typename std::remove_pointer_t<Func>>)

void link(Call call, Func funcName)

{

CodePtr<tag> function(funcName);

link(call, function);

}

template<PtrTag tag>

void link(Call call, CodePtr<tag> function)

{

ASSERT(call.isFlagSet(Call::Linkable));

call.m_label = applyOffset(call.m_label);

MacroAssembler::linkCall(code(), call, function);

}

template<PtrTag tag>

void link(Call call, CodeLocationLabel<tag> label)

{

link(call, CodePtr<tag>(label));

}

template<PtrTag tag>

void link(Jump jump, CodeLocationLabel<tag> label)

{

jump.m_label = applyOffset(jump.m_label);

MacroAssembler::linkJump(code(), jump, label);

}

template<PtrTag tag>

void link(const JumpList& list, CodeLocationLabel<tag> label)

{

for (const Jump& jump : list.jumps())

link(jump, label);

}

void patch(DataLabelPtr label, void* value)

{

AssemblerLabel target = applyOffset(label.m_label);

MacroAssembler::linkPointer(code(), target, value);

}

template<PtrTag tag>

void patch(DataLabelPtr label, CodeLocationLabel<tag> value)

{

AssemblerLabel target = applyOffset(label.m_label);

MacroAssembler::linkPointer(code(), target, value);

}

// These methods are used to obtain handles to allow the code to be relinked / repatched later.

template<PtrTag tag>

CodeLocationLabel<tag> entrypoint() const

{

return CodeLocationLabel<tag>(tagCodePtr<tag>(code()));

}

template<PtrTag tag>

CodeLocationCall<tag> locationOf(Call call)

{

ASSERT(call.isFlagSet(Call::Linkable));

ASSERT(!call.isFlagSet(Call::Near));

return CodeLocationCall<tag>(getLinkerAddress<tag>(applyOffset(call.m_label)));

}

template<PtrTag tag>

CodeLocationNearCall<tag> locationOfNearCall(Call call)

{

ASSERT(call.isFlagSet(Call::Linkable));

ASSERT(call.isFlagSet(Call::Near));

return CodeLocationNearCall<tag>(getLinkerAddress<tag>(applyOffset(call.m_label)),

call.isFlagSet(Call::Tail) ? NearCallMode::Tail : NearCallMode::Regular);

}

template<PtrTag tag>

CodeLocationLabel<tag> locationOf(PatchableJump jump)

{

return CodeLocationLabel<tag>(getLinkerAddress<tag>(applyOffset(jump.m_jump.m_label)));

}

template<PtrTag tag>

CodeLocationLabel<tag> locationOf(Label label)

{

return CodeLocationLabel<tag>(getLinkerAddress<tag>(applyOffset(label.m_label)));

}

template<PtrTag tag>

CodeLocationDataLabelPtr<tag> locationOf(DataLabelPtr label)

{

return CodeLocationDataLabelPtr<tag>(getLinkerAddress<tag>(applyOffset(label.m_label)));

}

template<PtrTag tag>

CodeLocationDataLabel32<tag> locationOf(DataLabel32 label)

{

return CodeLocationDataLabel32<tag>(getLinkerAddress<tag>(applyOffset(label.m_label)));

}

template<PtrTag tag>

CodeLocationDataLabelCompact<tag> locationOf(DataLabelCompact label)

{

return CodeLocationDataLabelCompact<tag>(getLinkerAddress<tag>(applyOffset(label.m_label)));

}

template<PtrTag tag>

CodeLocationConvertibleLoad<tag> locationOf(ConvertibleLoadLabel label)

{

return CodeLocationConvertibleLoad<tag>(getLinkerAddress<tag>(applyOffset(label.m_label)));

}

// This method obtains the return address of the call, given as an offset from

// the start of the code.

unsigned returnAddressOffset(Call call)

{

call.m_label = applyOffset(call.m_label);

return MacroAssembler::getLinkerCallReturnOffset(call);

}

uint32_t offsetOf(Label label)

{

return applyOffset(label.m_label).offset();

}

unsigned offsetOf(PatchableJump jump)

{

return applyOffset(jump.m_jump.m_label).offset();

}

// Upon completion of all patching 'FINALIZE_CODE()' should be called once to

// complete generation of the code. Alternatively, call

// finalizeCodeWithoutDisassembly() directly if you have your own way of

// displaying disassembly.

template<PtrTag tag>

CodeRef<tag> finalizeCodeWithoutDisassembly(ASCIILiteral simpleName)

{

return finalizeCodeWithoutDisassemblyImpl(simpleName).template retagged<tag>();

}

template<PtrTag tag, typename... Args>

CodeRef<tag> finalizeCodeWithDisassembly(bool dumpDisassembly, ASCIILiteral simpleName, const char* format, Args... args)

{

IGNORE_WARNINGS_BEGIN("format-security")

return finalizeCodeWithDisassemblyImpl(dumpDisassembly, simpleName, format, args...).template retagged<tag>();

IGNORE_WARNINGS_END

}

template<PtrTag tag>

CodePtr<tag> trampolineAt(Label label)

{

return CodePtr<tag>(MacroAssembler::AssemblerType_T::getRelocatedAddress(code(), applyOffset(label.m_label)));

}

void* debugAddress()

{

return m_code.dataLocation();

}

size_t NODELETE size() const { return m_size; }

bool wasAlreadyDisassembled() const { return m_alreadyDisassembled; }

void didAlreadyDisassemble() { m_alreadyDisassembled = true; }

JS_EXPORT_PRIVATE static void NODELETE clearProfileStatistics();

JS_EXPORT_PRIVATE static void dumpProfileStatistics(std::optional<PrintStream*> = std::nullopt);

void setIsThunk() { m_isThunk = true; }

void setIRDumpDebugInfo(std::unique_ptr<IRDumpDebugInfo>&& info) { m_irDumpDebugInfo = WTF::move(info); }

void setSourceCodeDumpDebugInfo(std::unique_ptr<SourceCodeDumpDebugInfo>&& info) { m_sourceCodeDebugInfo = WTF::move(info); }

JS_EXPORT_PRIVATE CodeRef<LinkBufferPtrTag> finalizeCodeWithoutDisassemblyImpl(ASCIILiteral);

JS_EXPORT_PRIVATE CodeRef<LinkBufferPtrTag> finalizeCodeWithDisassemblyImpl(bool dumpDisassembly, ASCIILiteral, const char* format, ...) WTF_ATTRIBUTE_PRINTF(4, 5);

#if ENABLE(BRANCH_COMPACTION)

int executableOffsetFor(int location)

{

// Returning 0 in this case works because at location <

// sizeof(int32_t), no compaction could have happened before this

// point as the assembler could not have placed a branch instruction

// within this space that required compaction.

if (location < static_cast<int>(sizeof(int32_t)))

return 0;

return std::bit_cast<int32_t*>(m_assemblerStorage.buffer())[location / sizeof(int32_t) - 1];

}

#endif

template <typename T> T applyOffset(T src)

{

#if ENABLE(BRANCH_COMPACTION)

src = src.labelAtOffset(-executableOffsetFor(src.offset()));

#endif

return src;

}

// Keep this private! - the underlying code should only be obtained externally via finalizeCode().

void* code() const

{

return m_code.dataLocation();

}

void linkComments(MacroAssembler&);

void allocate(MacroAssembler&, JITCompilationEffort);

template<PtrTag tag, typename T>

void* getLinkerAddress(T src)

{

void *code = this->code();

void* address = MacroAssembler::getLinkerAddress<tag>(code, src);

RELEASE_ASSERT(code <= untagCodePtr<tag>(address) && untagCodePtr<tag>(address) <= static_cast<char*>(code) + size());

return address;

}

JS_EXPORT_PRIVATE void linkCode(MacroAssembler&, JITCompilationEffort);

#if ENABLE(BRANCH_COMPACTION)

template <typename InstructionType>

void copyCompactAndLinkCode(MacroAssembler&, JITCompilationEffort);

#endif

void performFinalization();

#if DUMP_LINK_STATISTICS

static void dumpLinkStatistics(void* code, size_t initialSize, size_t finalSize);

#endif

#if DUMP_CODE

static void dumpCode(void* code, size_t);

#endif

void logJITCodeForJITDump(CodeRef<LinkBufferPtrTag>&, ASCIILiteral);

RefPtr<ExecutableMemoryHandle> m_executableMemory;

size_t m_size { 0 };

void* m_ownerUID { nullptr };

#if ENABLE(BRANCH_COMPACTION)

AssemblerData m_assemblerStorage;

#if ENABLE(JIT_SIGN_ASSEMBLER_BUFFER)

AssemblerHashes m_assemblerHashesStorage;

#endif

bool m_shouldPerformBranchCompaction { true };

#endif

bool m_didAllocate { false };

#ifndef NDEBUG

bool m_completed { false };

#endif

#if ASSERT_ENABLED

bool m_isJumpIsland { false };

#endif

bool m_alreadyDisassembled { false };

bool m_isThunk { false };

CacheFlushOnFinalize m_cacheFlushOnFinalize { CacheFlushOnFinalize::Yes };

bool m_isRewriting { false };

Profile m_profile { Profile::Uncategorized };

CodePtr<LinkBufferPtrTag> m_code;

Vector<Ref<SharedTask<void(LinkBuffer&)>>> m_linkTasks;

Vector<Ref<SharedTask<void(LinkBuffer&)>>> m_lateLinkTasks;

std::unique_ptr<IRDumpDebugInfo> m_irDumpDebugInfo;

std::unique_ptr<SourceCodeDumpDebugInfo> m_sourceCodeDebugInfo;

static size_t s_profileCummulativeLinkedSizes[numberOfProfiles];

static size_t s_profileCummulativeLinkedCounts[numberOfProfiles];