1 // Copyright (C) 2023 The Android Open Source Project
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 // http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14 //
15
16 #include "berberis/code_gen_lib/gen_wrapper.h"
17
18 #include "berberis/assembler/machine_code.h"
19 #include "berberis/assembler/x86_64.h"
20 #include "berberis/base/checks.h"
21 #include "berberis/guest_abi/guest_arguments.h"
22 #include "berberis/guest_state/guest_addr.h"
23 #include "berberis/intrinsics/macro_assembler.h"
24 #include "berberis/runtime_primitives/host_code.h"
25 #include "berberis/runtime_primitives/platform.h"
26
27 namespace berberis {
28
29 using x86_64::Assembler;
30
31 namespace {
32
ExtendIntArg(MacroAssembler<Assembler> & as,char type,Assembler::Register dst,Assembler::Register src)33 void ExtendIntArg(MacroAssembler<Assembler>& as,
34 char type,
35 Assembler::Register dst,
36 Assembler::Register src) {
37 if (type == 'z') {
38 as.Movzxbq(dst, src);
39 } else if (type == 'b') {
40 as.Movsxbq(dst, src);
41 } else if (type == 's') {
42 as.Movsxwq(dst, src);
43 } else if (type == 'c') {
44 as.Movzxwq(dst, src);
45 } else if (type == 'i') {
46 as.Movsxlq(dst, src);
47 } else if (type == 'p' || type == 'l') {
48 // Do nothing. The parameter is already 64 bits.
49 } else {
50 FATAL("non-integer signature char '%c'", type);
51 }
52 }
53
54 } // namespace
55
GenWrapGuestFunction(MachineCode * mc,GuestAddr pc,const char * signature,HostCode guest_runner,const char * name)56 void GenWrapGuestFunction(MachineCode* mc,
57 GuestAddr pc,
58 const char* signature,
59 HostCode guest_runner,
60 const char* name) {
61 UNUSED(name);
62
63 MacroAssembler<Assembler> as(mc);
64
65 // On function entry, rsp + 8 is a multiple of 16.
66 // Right before next function call, rsp is a multiple of 16.
67
68 // Default prologue.
69 as.Push(Assembler::rbp);
70 as.Movq(Assembler::rbp, Assembler::rsp);
71
72 static_assert(alignof(GuestArgumentBuffer) <= 16, "unexpected GuestArgumentBuffer alignment");
73
74 // Estimate guest argument buffer size.
75 // Each argument can be 2 8-bytes at most. Result can be 2 8-bytes at most.
76 // At least 8 arguments go to registers in GuestArgumentBuffer.
77 // First 8-byte of stack is in GuestArgumentBuffer.
78 // Result is return on registers in GuestArgumentBuffer.
79 // TODO(eaeltsin): maybe run parameter passing to calculate exactly?
80 size_t num_args = strlen(signature) - 1;
81 size_t max_stack_argv_size = (num_args > 8 ? num_args - 8 : 0) * 16;
82 size_t guest_argument_buffer_size = sizeof(GuestArgumentBuffer) - 8 + max_stack_argv_size;
83
84 size_t aligned_frame_size = AlignUp(guest_argument_buffer_size, 16);
85
86 // Allocate stack frame.
87 as.Subq(Assembler::rsp, static_cast<int32_t>(aligned_frame_size));
88
89 // rsp is 16-bytes aligned and points to GuestArgumentBuffer.
90
91 constexpr int kArgcOffset = offsetof(GuestArgumentBuffer, argc);
92 constexpr int kRescOffset = offsetof(GuestArgumentBuffer, resc);
93 constexpr int kArgvOffset = offsetof(GuestArgumentBuffer, argv);
94 constexpr int kFpArgcOffset = offsetof(GuestArgumentBuffer, fp_argc);
95 constexpr int kFpRescOffset = offsetof(GuestArgumentBuffer, fp_resc);
96 constexpr int kFpArgvOffset = offsetof(GuestArgumentBuffer, fp_argv);
97 constexpr int kStackArgcOffset = offsetof(GuestArgumentBuffer, stack_argc);
98 constexpr int kStackArgvOffset = offsetof(GuestArgumentBuffer, stack_argv);
99
100 const int params_offset = aligned_frame_size + 16;
101
102 // Convert parameters and set argc.
103 int argc = 0;
104 int fp_argc = 0;
105 int stack_argc = 0;
106 int host_stack_argc = 0;
107 static constexpr int kGuestParamRegs = 8;
108 static constexpr Assembler::Register kParamRegs[] = {
109 Assembler::rdi,
110 Assembler::rsi,
111 Assembler::rdx,
112 Assembler::rcx,
113 Assembler::r8,
114 Assembler::r9,
115 };
116 static constexpr Assembler::XMMRegister kFpParamRegs[] = {
117 Assembler::xmm0,
118 Assembler::xmm1,
119 Assembler::xmm2,
120 Assembler::xmm3,
121 Assembler::xmm4,
122 Assembler::xmm5,
123 Assembler::xmm6,
124 Assembler::xmm7,
125 };
126 for (size_t i = 1; signature[i] != '\0'; ++i) {
127 if (signature[i] == 'z' || signature[i] == 'b' || signature[i] == 's' || signature[i] == 'c' ||
128 signature[i] == 'i' || signature[i] == 'p' || signature[i] == 'l') {
129 if (argc < static_cast<int>(std::size(kParamRegs))) {
130 ExtendIntArg(as, signature[i], kParamRegs[argc], kParamRegs[argc]);
131 as.Movq({.base = Assembler::rsp, .disp = kArgvOffset + argc * 8}, kParamRegs[argc]);
132 } else if (argc < kGuestParamRegs) {
133 as.Movq(Assembler::rax,
134 {.base = Assembler::rsp, .disp = params_offset + host_stack_argc * 8});
135 ++host_stack_argc;
136 ExtendIntArg(as, signature[i], Assembler::rax, Assembler::rax);
137 as.Movq({.base = Assembler::rsp, .disp = kArgvOffset + argc * 8}, Assembler::rax);
138 } else {
139 as.Movq(Assembler::rax,
140 {.base = Assembler::rsp, .disp = params_offset + host_stack_argc * 8});
141 ++host_stack_argc;
142 ExtendIntArg(as, signature[i], Assembler::rax, Assembler::rax);
143 as.Movq({.base = Assembler::rsp, .disp = kStackArgvOffset + stack_argc * 8},
144 Assembler::rax);
145 ++stack_argc;
146 }
147 ++argc;
148 } else if (signature[i] == 'f' || signature[i] == 'd') {
149 // Floating-point parameters are passed in the floating-point parameter registers (fa0..7)
150 // first, then the general-purpose parameter registers (a0..7), then on the stack.
151 if (fp_argc < static_cast<int>(std::size(kFpParamRegs))) {
152 if (signature[i] == 'f') {
153 // LP64D requires 32-bit floats to be NaN boxed.
154 if (host_platform::kHasAVX) {
155 as.MacroNanBoxAVX<intrinsics::Float32>(kFpParamRegs[fp_argc], kFpParamRegs[fp_argc]);
156 } else {
157 as.MacroNanBox<intrinsics::Float32>(kFpParamRegs[fp_argc]);
158 }
159 }
160 if (host_platform::kHasAVX) {
161 as.Vmovq({.base = Assembler::rsp, .disp = kFpArgvOffset + fp_argc * 8},
162 kFpParamRegs[fp_argc]);
163 } else {
164 as.Movq({.base = Assembler::rsp, .disp = kFpArgvOffset + fp_argc * 8},
165 kFpParamRegs[fp_argc]);
166 }
167 } else if (argc < kGuestParamRegs) {
168 as.Movq(Assembler::rax,
169 {.base = Assembler::rsp, .disp = params_offset + host_stack_argc * 8});
170 ++host_stack_argc;
171 as.Movq({.base = Assembler::rsp, .disp = kArgvOffset + argc * 8}, Assembler::rax);
172 ++argc;
173 } else {
174 as.Movq(Assembler::rax,
175 {.base = Assembler::rsp, .disp = params_offset + host_stack_argc * 8});
176 ++host_stack_argc;
177 as.Movq({.base = Assembler::rsp, .disp = kStackArgvOffset + stack_argc * 8},
178 Assembler::rax);
179 ++stack_argc;
180 }
181 ++fp_argc;
182 } else {
183 FATAL("signature char '%c' not supported", signature[i]);
184 }
185 }
186 as.Movl({.base = Assembler::rsp, .disp = kArgcOffset}, std::min(argc, 8));
187 as.Movl({.base = Assembler::rsp, .disp = kFpArgcOffset}, std::min(fp_argc, 8));
188 // ATTENTION: GuestArgumentBuffer::stack_argc is in bytes!
189 as.Movl({.base = Assembler::rsp, .disp = kStackArgcOffset}, stack_argc * 8);
190
191 // Set resc.
192 if (signature[0] == 'z' || signature[0] == 'b' || signature[0] == 's' || signature[0] == 'c' ||
193 signature[0] == 'i' || signature[0] == 'p' || signature[0] == 'l') {
194 as.Movl({.base = Assembler::rsp, .disp = kRescOffset}, 1);
195 as.Movl({.base = Assembler::rsp, .disp = kFpRescOffset}, 0);
196 } else if (signature[0] == 'f' || signature[0] == 'd') {
197 as.Movl({.base = Assembler::rsp, .disp = kRescOffset}, 0);
198 as.Movl({.base = Assembler::rsp, .disp = kFpRescOffset}, 1);
199 } else {
200 CHECK_EQ('v', signature[0]);
201 as.Movl({.base = Assembler::rsp, .disp = kRescOffset}, 0);
202 as.Movl({.base = Assembler::rsp, .disp = kFpRescOffset}, 0);
203 }
204
205 // Call guest runner.
206 as.Movq(Assembler::rdi, pc);
207 as.Movq(Assembler::rsi, Assembler::rsp);
208 as.Call(guest_runner);
209
210 // Get the result.
211 if (signature[0] == 'z' || signature[0] == 'b' || signature[0] == 's' || signature[0] == 'c' ||
212 signature[0] == 'i' || signature[0] == 'p' || signature[0] == 'l') {
213 // It is not necessary to unbox integer return values. The callee will truncate rax to only
214 // retrieve the bits appropriate for the return type.
215 as.Movq(Assembler::rax, {.base = Assembler::rsp, .disp = kArgvOffset});
216 } else if (signature[0] == 'f') {
217 // Only take the lower 32 bits of the result register because floats are 1-extended (NaN boxed)
218 // on LP64D.
219 if (host_platform::kHasAVX) {
220 as.Vmovd(Assembler::xmm0, {.base = Assembler::rsp, .disp = kFpArgvOffset});
221 } else {
222 as.Movd(Assembler::xmm0, {.base = Assembler::rsp, .disp = kFpArgvOffset});
223 }
224 } else if (signature[0] == 'd') {
225 if (host_platform::kHasAVX) {
226 as.Vmovq(Assembler::xmm0, {.base = Assembler::rsp, .disp = kFpArgvOffset});
227 } else {
228 as.Movq(Assembler::xmm0, {.base = Assembler::rsp, .disp = kFpArgvOffset});
229 }
230 } else {
231 CHECK_EQ('v', signature[0]);
232 }
233
234 // Free stack frame.
235 as.Addq(Assembler::rsp, static_cast<int32_t>(aligned_frame_size));
236
237 // Default epilogue.
238 as.Pop(Assembler::rbp);
239 as.Ret();
240
241 as.Finalize();
242 }
243
244 } // namespace berberis
245