ydb/contrib/libs/llvm14/lib/ExecutionEngine/Orc/IndirectionUtils.cpp

//===---- IndirectionUtils.cpp - Utilities for call indirection in Orc ----===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//

#include "llvm/ExecutionEngine/Orc/IndirectionUtils.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/Triple.h"
#include "llvm/ExecutionEngine/JITLink/x86_64.h"
#include "llvm/ExecutionEngine/Orc/OrcABISupport.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/MC/MCDisassembler/MCDisassembler.h"
#include "llvm/MC/MCInstrAnalysis.h"
#include "llvm/Support/Format.h"
#include "llvm/Transforms/Utils/Cloning.h"
#include <sstream>

#define DEBUG_TYPE "orc"

using namespace llvm;
using namespace llvm::orc;

namespace {

class CompileCallbackMaterializationUnit : public orc::MaterializationUnit {
public:
  using CompileFunction = JITCompileCallbackManager::CompileFunction;

  CompileCallbackMaterializationUnit(SymbolStringPtr Name,
                                     CompileFunction Compile)
      : MaterializationUnit(Interface(
            SymbolFlagsMap({{Name, JITSymbolFlags::Exported}}), nullptr)),
        Name(std::move(Name)), Compile(std::move(Compile)) {}

  StringRef getName() const override { return "<Compile Callbacks>"; }

private:
  void materialize(std::unique_ptr<MaterializationResponsibility> R) override {
    SymbolMap Result;
    Result[Name] = JITEvaluatedSymbol(Compile(), JITSymbolFlags::Exported);
    // No dependencies, so these calls cannot fail.
    cantFail(R->notifyResolved(Result));
    cantFail(R->notifyEmitted());
  }

  void discard(const JITDylib &JD, const SymbolStringPtr &Name) override {
    llvm_unreachable("Discard should never occur on a LMU?");
  }

  SymbolStringPtr Name;
  CompileFunction Compile;
};

} // namespace

namespace llvm {
namespace orc {

TrampolinePool::~TrampolinePool() {}
void IndirectStubsManager::anchor() {}

Expected<JITTargetAddress>
JITCompileCallbackManager::getCompileCallback(CompileFunction Compile) {
  if (auto TrampolineAddr = TP->getTrampoline()) {
    auto CallbackName =
        ES.intern(std::string("cc") + std::to_string(++NextCallbackId));

    std::lock_guard<std::mutex> Lock(CCMgrMutex);
    AddrToSymbol[*TrampolineAddr] = CallbackName;
    cantFail(
        CallbacksJD.define(std::make_unique<CompileCallbackMaterializationUnit>(
            std::move(CallbackName), std::move(Compile))));
    return *TrampolineAddr;
  } else
    return TrampolineAddr.takeError();
}

JITTargetAddress JITCompileCallbackManager::executeCompileCallback(
    JITTargetAddress TrampolineAddr) {
  SymbolStringPtr Name;

  {
    std::unique_lock<std::mutex> Lock(CCMgrMutex);
    auto I = AddrToSymbol.find(TrampolineAddr);

    // If this address is not associated with a compile callback then report an
    // error to the execution session and return ErrorHandlerAddress to the
    // callee.
    if (I == AddrToSymbol.end()) {
      Lock.unlock();
      std::string ErrMsg;
      {
        raw_string_ostream ErrMsgStream(ErrMsg);
        ErrMsgStream << "No compile callback for trampoline at "
                     << format("0x%016" PRIx64, TrampolineAddr);
      }
      ES.reportError(
          make_error<StringError>(std::move(ErrMsg), inconvertibleErrorCode()));
      return ErrorHandlerAddress;
    } else
      Name = I->second;
  }

  if (auto Sym =
          ES.lookup(makeJITDylibSearchOrder(
                        &CallbacksJD, JITDylibLookupFlags::MatchAllSymbols),
                    Name))
    return Sym->getAddress();
  else {
    llvm::dbgs() << "Didn't find callback.\n";
    // If anything goes wrong materializing Sym then report it to the session
    // and return the ErrorHandlerAddress;
    ES.reportError(Sym.takeError());
    return ErrorHandlerAddress;
  }
}

Expected<std::unique_ptr<JITCompileCallbackManager>>
createLocalCompileCallbackManager(const Triple &T, ExecutionSession &ES,
                                  JITTargetAddress ErrorHandlerAddress) {
  switch (T.getArch()) {
  default:
    return make_error<StringError>(
        std::string("No callback manager available for ") + T.str(),
        inconvertibleErrorCode());
  case Triple::aarch64:
  case Triple::aarch64_32: {
    typedef orc::LocalJITCompileCallbackManager<orc::OrcAArch64> CCMgrT;
    return CCMgrT::Create(ES, ErrorHandlerAddress);
    }

    case Triple::x86: {
      typedef orc::LocalJITCompileCallbackManager<orc::OrcI386> CCMgrT;
      return CCMgrT::Create(ES, ErrorHandlerAddress);
    }

    case Triple::mips: {
      typedef orc::LocalJITCompileCallbackManager<orc::OrcMips32Be> CCMgrT;
      return CCMgrT::Create(ES, ErrorHandlerAddress);
    }
    case Triple::mipsel: {
      typedef orc::LocalJITCompileCallbackManager<orc::OrcMips32Le> CCMgrT;
      return CCMgrT::Create(ES, ErrorHandlerAddress);
    }

    case Triple::mips64:
    case Triple::mips64el: {
      typedef orc::LocalJITCompileCallbackManager<orc::OrcMips64> CCMgrT;
      return CCMgrT::Create(ES, ErrorHandlerAddress);
    }

    case Triple::x86_64: {
      if (T.getOS() == Triple::OSType::Win32) {
        typedef orc::LocalJITCompileCallbackManager<orc::OrcX86_64_Win32> CCMgrT;
        return CCMgrT::Create(ES, ErrorHandlerAddress);
      } else {
        typedef orc::LocalJITCompileCallbackManager<orc::OrcX86_64_SysV> CCMgrT;
        return CCMgrT::Create(ES, ErrorHandlerAddress);
      }
    }

  }
}

std::function<std::unique_ptr<IndirectStubsManager>()>
createLocalIndirectStubsManagerBuilder(const Triple &T) {
  switch (T.getArch()) {
    default:
      return [](){
        return std::make_unique<
                       orc::LocalIndirectStubsManager<orc::OrcGenericABI>>();
      };

    case Triple::aarch64:
    case Triple::aarch64_32:
      return [](){
        return std::make_unique<
                       orc::LocalIndirectStubsManager<orc::OrcAArch64>>();
      };

    case Triple::x86:
      return [](){
        return std::make_unique<
                       orc::LocalIndirectStubsManager<orc::OrcI386>>();
      };

    case Triple::mips:
      return [](){
          return std::make_unique<
                      orc::LocalIndirectStubsManager<orc::OrcMips32Be>>();
      };

    case Triple::mipsel:
      return [](){
          return std::make_unique<
                      orc::LocalIndirectStubsManager<orc::OrcMips32Le>>();
      };

    case Triple::mips64:
    case Triple::mips64el:
      return [](){
          return std::make_unique<
                      orc::LocalIndirectStubsManager<orc::OrcMips64>>();
      };

    case Triple::x86_64:
      if (T.getOS() == Triple::OSType::Win32) {
        return [](){
          return std::make_unique<
                     orc::LocalIndirectStubsManager<orc::OrcX86_64_Win32>>();
        };
      } else {
        return [](){
          return std::make_unique<
                     orc::LocalIndirectStubsManager<orc::OrcX86_64_SysV>>();
        };
      }

  }
}

Constant* createIRTypedAddress(FunctionType &FT, JITTargetAddress Addr) {
  Constant *AddrIntVal =
    ConstantInt::get(Type::getInt64Ty(FT.getContext()), Addr);
  Constant *AddrPtrVal =
    ConstantExpr::getCast(Instruction::IntToPtr, AddrIntVal,
                          PointerType::get(&FT, 0));
  return AddrPtrVal;
}

GlobalVariable* createImplPointer(PointerType &PT, Module &M,
                                  const Twine &Name, Constant *Initializer) {
  auto IP = new GlobalVariable(M, &PT, false, GlobalValue::ExternalLinkage,
                               Initializer, Name, nullptr,
                               GlobalValue::NotThreadLocal, 0, true);
  IP->setVisibility(GlobalValue::HiddenVisibility);
  return IP;
}

void makeStub(Function &F, Value &ImplPointer) {
  assert(F.isDeclaration() && "Can't turn a definition into a stub.");
  assert(F.getParent() && "Function isn't in a module.");
  Module &M = *F.getParent();
  BasicBlock *EntryBlock = BasicBlock::Create(M.getContext(), "entry", &F);
  IRBuilder<> Builder(EntryBlock);
  LoadInst *ImplAddr = Builder.CreateLoad(F.getType(), &ImplPointer);
  std::vector<Value*> CallArgs;
  for (auto &A : F.args())
    CallArgs.push_back(&A);
  CallInst *Call = Builder.CreateCall(F.getFunctionType(), ImplAddr, CallArgs);
  Call->setTailCall();
  Call->setAttributes(F.getAttributes());
  if (F.getReturnType()->isVoidTy())
    Builder.CreateRetVoid();
  else
    Builder.CreateRet(Call);
}

std::vector<GlobalValue *> SymbolLinkagePromoter::operator()(Module &M) {
  std::vector<GlobalValue *> PromotedGlobals;

  for (auto &GV : M.global_values()) {
    bool Promoted = true;

    // Rename if necessary.
    if (!GV.hasName())
      GV.setName("__orc_anon." + Twine(NextId++));
    else if (GV.getName().startswith("\01L"))
      GV.setName("__" + GV.getName().substr(1) + "." + Twine(NextId++));
    else if (GV.hasLocalLinkage())
      GV.setName("__orc_lcl." + GV.getName() + "." + Twine(NextId++));
    else
      Promoted = false;

    if (GV.hasLocalLinkage()) {
      GV.setLinkage(GlobalValue::ExternalLinkage);
      GV.setVisibility(GlobalValue::HiddenVisibility);
      Promoted = true;
    }
    GV.setUnnamedAddr(GlobalValue::UnnamedAddr::None);

    if (Promoted)
      PromotedGlobals.push_back(&GV);
  }

  return PromotedGlobals;
}

Function* cloneFunctionDecl(Module &Dst, const Function &F,
                            ValueToValueMapTy *VMap) {
  Function *NewF =
    Function::Create(cast<FunctionType>(F.getValueType()),
                     F.getLinkage(), F.getName(), &Dst);
  NewF->copyAttributesFrom(&F);

  if (VMap) {
    (*VMap)[&F] = NewF;
    auto NewArgI = NewF->arg_begin();
    for (auto ArgI = F.arg_begin(), ArgE = F.arg_end(); ArgI != ArgE;
         ++ArgI, ++NewArgI)
      (*VMap)[&*ArgI] = &*NewArgI;
  }

  return NewF;
}

void moveFunctionBody(Function &OrigF, ValueToValueMapTy &VMap,
                      ValueMaterializer *Materializer,
                      Function *NewF) {
  assert(!OrigF.isDeclaration() && "Nothing to move");
  if (!NewF)
    NewF = cast<Function>(VMap[&OrigF]);
  else
    assert(VMap[&OrigF] == NewF && "Incorrect function mapping in VMap.");
  assert(NewF && "Function mapping missing from VMap.");
  assert(NewF->getParent() != OrigF.getParent() &&
         "moveFunctionBody should only be used to move bodies between "
         "modules.");

  SmallVector<ReturnInst *, 8> Returns; // Ignore returns cloned.
  CloneFunctionInto(NewF, &OrigF, VMap,
                    CloneFunctionChangeType::DifferentModule, Returns, "",
                    nullptr, nullptr, Materializer);
  OrigF.deleteBody();
}

GlobalVariable* cloneGlobalVariableDecl(Module &Dst, const GlobalVariable &GV,
                                        ValueToValueMapTy *VMap) {
  GlobalVariable *NewGV = new GlobalVariable(
      Dst, GV.getValueType(), GV.isConstant(),
      GV.getLinkage(), nullptr, GV.getName(), nullptr,
      GV.getThreadLocalMode(), GV.getType()->getAddressSpace());
  NewGV->copyAttributesFrom(&GV);
  if (VMap)
    (*VMap)[&GV] = NewGV;
  return NewGV;
}

void moveGlobalVariableInitializer(GlobalVariable &OrigGV,
                                   ValueToValueMapTy &VMap,
                                   ValueMaterializer *Materializer,
                                   GlobalVariable *NewGV) {
  assert(OrigGV.hasInitializer() && "Nothing to move");
  if (!NewGV)
    NewGV = cast<GlobalVariable>(VMap[&OrigGV]);
  else
    assert(VMap[&OrigGV] == NewGV &&
           "Incorrect global variable mapping in VMap.");
  assert(NewGV->getParent() != OrigGV.getParent() &&
         "moveGlobalVariableInitializer should only be used to move "
         "initializers between modules");

  NewGV->setInitializer(MapValue(OrigGV.getInitializer(), VMap, RF_None,
                                 nullptr, Materializer));
}

GlobalAlias* cloneGlobalAliasDecl(Module &Dst, const GlobalAlias &OrigA,
                                  ValueToValueMapTy &VMap) {
  assert(OrigA.getAliasee() && "Original alias doesn't have an aliasee?");
  auto *NewA = GlobalAlias::create(OrigA.getValueType(),
                                   OrigA.getType()->getPointerAddressSpace(),
                                   OrigA.getLinkage(), OrigA.getName(), &Dst);
  NewA->copyAttributesFrom(&OrigA);
  VMap[&OrigA] = NewA;
  return NewA;
}

void cloneModuleFlagsMetadata(Module &Dst, const Module &Src,
                              ValueToValueMapTy &VMap) {
  auto *MFs = Src.getModuleFlagsMetadata();
  if (!MFs)
    return;
  for (auto *MF : MFs->operands())
    Dst.addModuleFlag(MapMetadata(MF, VMap));
}

Error addFunctionPointerRelocationsToCurrentSymbol(jitlink::Symbol &Sym,
                                                   jitlink::LinkGraph &G,
                                                   MCDisassembler &Disassembler,
                                                   MCInstrAnalysis &MIA) {
  // AArch64 appears to already come with the necessary relocations. Among other
  // architectures, only x86_64 is currently implemented here.
  if (G.getTargetTriple().getArch() != Triple::x86_64)
    return Error::success();

  raw_null_ostream CommentStream;
  auto &STI = Disassembler.getSubtargetInfo();

  // Determine the function bounds
  auto &B = Sym.getBlock();
  assert(!B.isZeroFill() && "expected content block");
  auto SymAddress = Sym.getAddress();
  auto SymStartInBlock =
      (const uint8_t *)B.getContent().data() + Sym.getOffset();
  auto SymSize = Sym.getSize() ? Sym.getSize() : B.getSize() - Sym.getOffset();
  auto Content = makeArrayRef(SymStartInBlock, SymSize);

  LLVM_DEBUG(dbgs() << "Adding self-relocations to " << Sym.getName() << "\n");

  SmallDenseSet<uintptr_t, 8> ExistingRelocations;
  for (auto &E : B.edges()) {
    if (E.isRelocation())
      ExistingRelocations.insert(E.getOffset());
  }

  size_t I = 0;
  while (I < Content.size()) {
    MCInst Instr;
    uint64_t InstrSize = 0;
    uint64_t InstrStart = SymAddress.getValue() + I;
    auto DecodeStatus = Disassembler.getInstruction(
        Instr, InstrSize, Content.drop_front(I), InstrStart, CommentStream);
    if (DecodeStatus != MCDisassembler::Success) {
      LLVM_DEBUG(dbgs() << "Aborting due to disassembly failure at address "
                        << InstrStart);
      return make_error<StringError>(
          formatv("failed to disassemble at address {0:x16}", InstrStart),
          inconvertibleErrorCode());
    }
    // Advance to the next instruction.
    I += InstrSize;

    // Check for a PC-relative address equal to the symbol itself.
    auto PCRelAddr =
        MIA.evaluateMemoryOperandAddress(Instr, &STI, InstrStart, InstrSize);
    if (!PCRelAddr || *PCRelAddr != SymAddress.getValue())
      continue;

    auto RelocOffInInstr =
        MIA.getMemoryOperandRelocationOffset(Instr, InstrSize);
    if (!RelocOffInInstr.hasValue() ||
        InstrSize - RelocOffInInstr.getValue() != 4) {
      LLVM_DEBUG(dbgs() << "Skipping unknown self-relocation at "
                        << InstrStart);
      continue;
    }

    auto RelocOffInBlock = orc::ExecutorAddr(InstrStart) + *RelocOffInInstr -
                           SymAddress + Sym.getOffset();
    if (ExistingRelocations.contains(RelocOffInBlock))
      continue;

    LLVM_DEBUG(dbgs() << "Adding delta32 self-relocation at " << InstrStart);
    B.addEdge(jitlink::x86_64::Delta32, RelocOffInBlock, Sym, /*Addend=*/-4);
  }
  return Error::success();
}

} // End namespace orc.
} // End namespace llvm.