android-15.0.0_r3/s

/*
 * Copyright (C) 2023 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "berberis/code_gen_lib/code_gen_lib.h"

#include "berberis/assembler/machine_code.h"
#include "berberis/assembler/x86_64.h"
#include "berberis/base/bit_util.h"
#include "berberis/base/checks.h"
#include "berberis/base/config.h"
#include "berberis/calling_conventions/calling_conventions_x86_64.h"
#include "berberis/code_gen_lib/gen_adaptor.h"
#include "berberis/guest_state/guest_addr.h"
#include "berberis/guest_state/guest_state.h"
#include "berberis/instrument/trampolines.h"
#include "berberis/kernel_api/run_guest_syscall.h"
#include "berberis/runtime_primitives/host_code.h"
#include "berberis/runtime_primitives/translation_cache.h"

namespace berberis {

namespace {

// Emits checks for pending emulated signals. If pending signals present
// generated code returns to the main dispatcher loop to handle them.
// To ensure we don't loop endlessly in generated code, it must be called on
// every region exit (loops within regions must be taken care of separately).
// Thus we call it in EmitDirectDispatch and EmitIndirectDispatch.
void EmitCheckSignalsAndMaybeReturn(x86_64::Assembler* as) {
  // C++:
  //   std::atomic_int_least8_t pending_signals_status;
  //   uint8_t status = pending_signals_status.load(std::memory_order_acquire);
  //   if (status == kPendingSignalsPresent) { ... }
  // x86_64 asm:
  //   cmpb pending_signals_status, kPendingSignalsPresent
  const size_t offset = offsetof(ThreadState, pending_signals_status);
  as->Cmpb({.base = x86_64::Assembler::rbp, .disp = offset}, kPendingSignalsPresent);
  as->Jcc(x86_64::Assembler::Condition::kEqual, kEntryExitGeneratedCode);
}

}  // namespace

void GenTrampolineAdaptor(MachineCode* mc,
                          GuestAddr pc,
                          HostCode marshall,
                          const void* callee,
                          const char* name) {
  x86_64::Assembler as(mc);

  // Update insn_addr to the current PC.  This way, code generated by this
  // function does not require insn_addr to be up to date upon entry.  Note that
  // the trampoline that we call requires insn_addr to be up to date.
  as.Movq(as.rdi, pc);
  as.Movq({.base = as.rbp, .disp = offsetof(ThreadState, cpu.insn_addr)}, as.rdi);
  as.Movq({.base = as.rbp, .disp = offsetof(ThreadState, residence)}, kOutsideGeneratedCode);

  if (kInstrumentTrampolines) {
    if (auto instrument = GetOnTrampolineCall(name)) {
      as.Movq(as.rdi, as.rbp);
      as.Movq(as.rsi, reinterpret_cast<intptr_t>(name));
      as.Call(AsHostCode(instrument));
    }
  }

  // void Trampoline(void*, ThreadState*);
  as.Movq(as.rdi, reinterpret_cast<intptr_t>(callee));
  as.Movq(as.rsi, as.rbp);
  as.Call(marshall);

  if (kInstrumentTrampolines) {
    if (auto instrument = GetOnTrampolineReturn(name)) {
      as.Movq(as.rdi, as.rbp);
      as.Movq(as.rsi, reinterpret_cast<intptr_t>(name));
      as.Call(AsHostCode(instrument));
    }
  }

  // j ra
  // Prefer rdx, since rax/rcx will result in extra moves inside EmitIndirectDispatch.
  as.Movq(as.rdx, {.base = as.rbp, .disp = offsetof(ThreadState, cpu.x[RA])});
  // We are returning to generated code.
  as.Movq({.base = as.rbp, .disp = offsetof(ThreadState, residence)}, kInsideGeneratedCode);
  EmitIndirectDispatch(&as, as.rdx);
  as.Finalize();
}

void EmitSyscall(x86_64::Assembler* as, GuestAddr pc) {
  // We may run guest signal handler during the syscall so insn_addr needs to be synched.
  as->Movq(as->rdi, pc);
  as->Movq({.base = as->rbp, .disp = offsetof(ThreadState, cpu.insn_addr)}, as->rdi);
  as->Movq({.base = as->rbp, .disp = offsetof(ThreadState, residence)}, kOutsideGeneratedCode);

  // void RunGuestSyscall(ThreadState*);
  as->Movq(as->rdi, as->rbp);
  as->Call(AsHostCode(RunGuestSyscall));

  // We are returning to generated code.
  as->Movq({.base = as->rbp, .disp = offsetof(ThreadState, residence)}, kInsideGeneratedCode);
  // Advance to the next instruction.
  // TODO(b/161722184): if syscall is interrupted by signal, signal handler might overwrite the
  // insn_addr, so incrementing insn_addr here may be incorrect. This problem also exists in the
  // interpreter. On the other hand syscalls can only be interruprted by asynchroneous signals which
  // are unlikely to overwrite insn_addr.
  EmitDirectDispatch(as, pc + 4, /*check_pending_signals=*/true);
}

void EmitDirectDispatch(x86_64::Assembler* as, GuestAddr pc, bool check_pending_signals) {
  // insn_addr is passed between regions in rax.
  as->Movq(as->rax, pc);

  if (!config::kLinkJumpsBetweenRegions) {
    as->Jmp(kEntryExitGeneratedCode);
    return;
  }

  if (check_pending_signals) {
    EmitCheckSignalsAndMaybeReturn(as);
  }

  CHECK_EQ(pc & 0xffff000000000000, 0);
  as->Movq(as->rcx,
           reinterpret_cast<uint64_t>(TranslationCache::GetInstance()->GetHostCodePtr(pc)));
  as->Jmpq({.base = as->rcx});
}

void EmitExitGeneratedCode(x86_64::Assembler* as, x86_64::Assembler::Register target) {
  // insn_addr is passed between regions in rax.
  if (target != as->rax) {
    as->Movq(as->rax, target);
  }

  as->Jmp(kEntryExitGeneratedCode);
}

void EmitIndirectDispatch(x86_64::Assembler* as, x86_64::Assembler::Register target) {
  // insn_addr is passed between regions in rax.
  as->Movq(as->rax, target);

  if (!config::kLinkJumpsBetweenRegions) {
    as->Jmp(kEntryExitGeneratedCode);
    return;
  }

  // rax and rcx are used as scratches.
  if (target == as->rax || target == as->rcx) {
    as->Movq(as->rdx, target);
    target = as->rdx;
  }

  EmitCheckSignalsAndMaybeReturn(as);

  auto main_table_ptr = TranslationCache::GetInstance()->main_table_ptr();

  as->Shrq(as->rax, int8_t{24});
  as->Andl(as->rax, 0xffffff);
  as->Movq(as->rcx, reinterpret_cast<uint64_t>(main_table_ptr));
  as->Movq(as->rcx, {.base = as->rcx, .index = as->rax, .scale = x86_64::Assembler::kTimesEight});

  as->Movq(as->rax, target);
  as->Andq(as->rax, 0xffffff);
  as->Movq(as->rcx, {.base = as->rcx, .index = as->rax, .scale = x86_64::Assembler::kTimesEight});

  // insn_addr is passed between regions in rax.
  as->Movq(as->rax, target);

  as->Jmp(as->rcx);
}

void EmitAllocStackFrame(x86_64::Assembler* as, uint32_t frame_size) {
  if (frame_size > config::kFrameSizeAtTranslatedCode) {
    uint32_t extra_size = AlignUp(frame_size - config::kFrameSizeAtTranslatedCode,
                                  x86_64::CallingConventions::kStackAlignmentBeforeCall);
    as->Subq(as->rsp, extra_size);
  }
}

void EmitFreeStackFrame(x86_64::Assembler* as, uint32_t frame_size) {
  if (frame_size > config::kFrameSizeAtTranslatedCode) {
    uint32_t extra_size = AlignUp(frame_size - config::kFrameSizeAtTranslatedCode,
                                  x86_64::CallingConventions::kStackAlignmentBeforeCall);
    as->Addq(as->rsp, extra_size);
  }
}
}  // namespace berberis