1 /*
2 * Copyright (C) 2023 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include <cstdint>
18
19 #include "lite_translator.h"
20
21 #include "berberis/base/checks.h"
22 #include "berberis/base/macros.h"
23 #include "berberis/code_gen_lib/code_gen_lib.h"
24 #include "berberis/decoder/riscv64/decoder.h"
25
26 namespace berberis {
27
28 using Register = LiteTranslator::Register;
29 using Condition = LiteTranslator::Condition;
30
31 //
32 // Instruction implementations.
33 //
34
Op(Decoder::OpOpcode opcode,Register arg1,Register arg2)35 Register LiteTranslator::Op(Decoder::OpOpcode opcode, Register arg1, Register arg2) {
36 using OpOpcode = Decoder::OpOpcode;
37 Register res = AllocTempReg();
38 switch (opcode) {
39 case OpOpcode::kAdd:
40 as_.Movq(res, arg1);
41 as_.Addq(res, arg2);
42 break;
43 case OpOpcode::kSub:
44 as_.Movq(res, arg1);
45 as_.Subq(res, arg2);
46 break;
47 case OpOpcode::kAnd:
48 as_.Movq(res, arg1);
49 as_.Andq(res, arg2);
50 break;
51 case OpOpcode::kOr:
52 as_.Movq(res, arg1);
53 as_.Orq(res, arg2);
54 break;
55 case OpOpcode::kXor:
56 as_.Movq(res, arg1);
57 as_.Xorq(res, arg2);
58 break;
59 case OpOpcode::kSll:
60 case OpOpcode::kSrl:
61 case OpOpcode::kSra:
62 as_.Movq(res, arg1);
63 as_.Movq(as_.rcx, arg2);
64 if (opcode == OpOpcode::kSrl) {
65 as_.ShrqByCl(res);
66 } else if (opcode == OpOpcode::kSll) {
67 as_.ShlqByCl(res);
68 } else if (opcode == OpOpcode::kSra) {
69 as_.SarqByCl(res);
70 } else {
71 FATAL("Unexpected OpOpcode");
72 }
73 break;
74 case OpOpcode::kSlt:
75 as_.Xorq(res, res);
76 as_.Cmpq(arg1, arg2);
77 as_.Setcc(Condition::kLess, res);
78 break;
79 case OpOpcode::kSltu:
80 as_.Xorq(res, res);
81 as_.Cmpq(arg1, arg2);
82 as_.Setcc(Condition::kBelow, res);
83 break;
84 case OpOpcode::kMul:
85 as_.Movq(res, arg1);
86 as_.Imulq(res, arg2);
87 break;
88 case OpOpcode::kMulh:
89 as_.Movq(res, arg1);
90 as_.Movq(as_.rax, arg1);
91 as_.Imulq(arg2);
92 as_.Movq(res, as_.rdx);
93 break;
94 case OpOpcode::kMulhsu: {
95 as_.Movq(res, arg1);
96 as_.Movq(as_.rax, arg2);
97 as_.Mulq(res);
98 as_.Sarq(res, int8_t{63});
99 as_.Imulq(res, arg2);
100 as_.Addq(res, as_.rdx);
101 break;
102 }
103 case OpOpcode::kMulhu:
104 as_.Movq(as_.rax, arg1);
105 as_.Mulq(arg2);
106 as_.Movq(res, as_.rdx);
107 break;
108 case Decoder::OpOpcode::kAndn:
109 if (host_platform::kHasBMI) {
110 as_.Andnq(res, arg2, arg1);
111 } else {
112 as_.Movq(res, arg2);
113 as_.Notq(res);
114 as_.Andq(res, arg1);
115 }
116 break;
117 case Decoder::OpOpcode::kOrn:
118 as_.Movq(res, arg2);
119 as_.Notq(res);
120 as_.Orq(res, arg1);
121 break;
122 case Decoder::OpOpcode::kXnor:
123 as_.Movq(res, arg2);
124 as_.Xorq(res, arg1);
125 as_.Notq(res);
126 break;
127 default:
128 Undefined();
129 return {};
130 }
131 return res;
132 }
133
Op32(Decoder::Op32Opcode opcode,Register arg1,Register arg2)134 Register LiteTranslator::Op32(Decoder::Op32Opcode opcode, Register arg1, Register arg2) {
135 using Op32Opcode = Decoder::Op32Opcode;
136 Register res = AllocTempReg();
137 switch (opcode) {
138 case Op32Opcode::kAddw:
139 as_.Movl(res, arg1);
140 as_.Addl(res, arg2);
141 as_.Movsxlq(res, res);
142 break;
143 case Op32Opcode::kSubw:
144 as_.Movl(res, arg1);
145 as_.Subl(res, arg2);
146 as_.Movsxlq(res, res);
147 break;
148 case Op32Opcode::kSllw:
149 case Op32Opcode::kSrlw:
150 case Op32Opcode::kSraw:
151 as_.Movl(res, arg1);
152 as_.Movl(as_.rcx, arg2);
153 if (opcode == Op32Opcode::kSrlw) {
154 as_.ShrlByCl(res);
155 } else if (opcode == Op32Opcode::kSllw) {
156 as_.ShllByCl(res);
157 } else if (opcode == Op32Opcode::kSraw) {
158 as_.SarlByCl(res);
159 } else {
160 FATAL("Unexpected Op32Opcode");
161 }
162 as_.Movsxlq(res, res);
163 break;
164 case Op32Opcode::kMulw:
165 as_.Movl(res, arg1);
166 as_.Imull(res, arg2);
167 as_.Movsxlq(res, res);
168 break;
169 default:
170 Undefined();
171 return {};
172 }
173 return res;
174 }
175
OpImm(Decoder::OpImmOpcode opcode,Register arg,int16_t imm)176 Register LiteTranslator::OpImm(Decoder::OpImmOpcode opcode, Register arg, int16_t imm) {
177 using OpImmOpcode = Decoder::OpImmOpcode;
178 Register res = AllocTempReg();
179 switch (opcode) {
180 case OpImmOpcode::kAddi:
181 as_.Movq(res, arg);
182 as_.Addq(res, imm);
183 break;
184 case OpImmOpcode::kSlti:
185 as_.Xorq(res, res);
186 as_.Cmpq(arg, imm);
187 as_.Setcc(Condition::kLess, res);
188 break;
189 case OpImmOpcode::kSltiu:
190 as_.Xorq(res, res);
191 as_.Cmpq(arg, imm);
192 as_.Setcc(Condition::kBelow, res);
193 break;
194 case OpImmOpcode::kXori:
195 as_.Movq(res, arg);
196 as_.Xorq(res, imm);
197 break;
198 case OpImmOpcode::kOri:
199 as_.Movq(res, arg);
200 as_.Orq(res, imm);
201 break;
202 case OpImmOpcode::kAndi:
203 as_.Movq(res, arg);
204 as_.Andq(res, imm);
205 break;
206 default:
207 Undefined();
208 return {};
209 }
210 return res;
211 }
212
OpImm32(Decoder::OpImm32Opcode opcode,Register arg,int16_t imm)213 Register LiteTranslator::OpImm32(Decoder::OpImm32Opcode opcode, Register arg, int16_t imm) {
214 Register res = AllocTempReg();
215 switch (opcode) {
216 case Decoder::OpImm32Opcode::kAddiw:
217 as_.Movl(res, arg);
218 as_.Addl(res, imm);
219 as_.Movsxlq(res, res);
220 break;
221 default:
222 Undefined();
223 return {};
224 }
225 return res;
226 }
227
Slli(Register arg,int8_t imm)228 Register LiteTranslator::Slli(Register arg, int8_t imm) {
229 Register res = AllocTempReg();
230 as_.Movq(res, arg);
231 as_.Shlq(res, imm);
232 return res;
233 }
234
Srli(Register arg,int8_t imm)235 Register LiteTranslator::Srli(Register arg, int8_t imm) {
236 Register res = AllocTempReg();
237 as_.Movq(res, arg);
238 as_.Shrq(res, imm);
239 return res;
240 }
241
Srai(Register arg,int8_t imm)242 Register LiteTranslator::Srai(Register arg, int8_t imm) {
243 Register res = AllocTempReg();
244 as_.Movq(res, arg);
245 as_.Sarq(res, imm);
246 return res;
247 }
248
ShiftImm32(Decoder::ShiftImm32Opcode opcode,Register arg,uint16_t imm)249 Register LiteTranslator::ShiftImm32(Decoder::ShiftImm32Opcode opcode, Register arg, uint16_t imm) {
250 using ShiftImm32Opcode = Decoder::ShiftImm32Opcode;
251 Register res = AllocTempReg();
252 as_.Movl(res, arg);
253 as_.Movl(as_.rcx, imm);
254 if (opcode == ShiftImm32Opcode::kSrliw) {
255 as_.ShrlByCl(res);
256 } else if (opcode == ShiftImm32Opcode::kSlliw) {
257 as_.ShllByCl(res);
258 } else if (opcode == ShiftImm32Opcode::kSraiw) {
259 as_.SarlByCl(res);
260 } else {
261 Undefined();
262 return {};
263 }
264 as_.Movsxlq(res, res);
265 return res;
266 }
267
Rori(Register arg,int8_t shamt)268 Register LiteTranslator::Rori(Register arg, int8_t shamt) {
269 Register res = AllocTempReg();
270 as_.Movq(res, arg);
271 as_.Rorq(res, shamt);
272 return res;
273 }
274
Roriw(Register arg,int8_t shamt)275 Register LiteTranslator::Roriw(Register arg, int8_t shamt) {
276 Register res = AllocTempReg();
277 as_.Movq(res, arg);
278 as_.Rorl(res, shamt);
279 as_.Movsxlq(res, res);
280 return res;
281 }
282
Lui(int32_t imm)283 Register LiteTranslator::Lui(int32_t imm) {
284 Register res = AllocTempReg();
285 as_.Movq(res, imm);
286 return res;
287 }
288
Auipc(int32_t imm)289 Register LiteTranslator::Auipc(int32_t imm) {
290 Register res = GetImm(GetInsnAddr());
291 as_.Addq(res, imm);
292 return res;
293 }
294
CompareAndBranch(Decoder::BranchOpcode opcode,Register arg1,Register arg2,int16_t offset)295 void LiteTranslator::CompareAndBranch(Decoder::BranchOpcode opcode,
296 Register arg1,
297 Register arg2,
298 int16_t offset) {
299 Assembler::Label* cont = as_.MakeLabel();
300 as_.Cmpq(arg1, arg2);
301 switch (opcode) {
302 case Decoder::BranchOpcode::kBeq:
303 as_.Jcc(Condition::kNotEqual, *cont);
304 break;
305 case Decoder::BranchOpcode::kBne:
306 as_.Jcc(Condition::kEqual, *cont);
307 break;
308 case Decoder::BranchOpcode::kBltu:
309 as_.Jcc(Condition::kAboveEqual, *cont);
310 break;
311 case Decoder::BranchOpcode::kBgeu:
312 as_.Jcc(Condition::kBelow, *cont);
313 break;
314 case Decoder::BranchOpcode::kBlt:
315 as_.Jcc(Condition::kGreaterEqual, *cont);
316 break;
317 case Decoder::BranchOpcode::kBge:
318 as_.Jcc(Condition::kLess, *cont);
319 break;
320 default:
321 return Undefined();
322 }
323 ExitRegion(GetInsnAddr() + offset);
324 as_.Bind(cont);
325 }
326
ExitGeneratedCode(GuestAddr target)327 void LiteTranslator::ExitGeneratedCode(GuestAddr target) {
328 StoreMappedRegs();
329 // EmitExitGeneratedCode is more efficient if receives target in rax.
330 as_.Movq(as_.rax, target);
331 EmitExitGeneratedCode(&as_, as_.rax);
332 }
333
ExitRegion(GuestAddr target)334 void LiteTranslator::ExitRegion(GuestAddr target) {
335 StoreMappedRegs();
336 if (params_.allow_dispatch) {
337 EmitDirectDispatch(&as_, target, /* check_pending_signals */ true);
338 } else {
339 // EmitExitGeneratedCode is more efficient if receives target in rax.
340 as_.Movq(as_.rax, target);
341 EmitExitGeneratedCode(&as_, as_.rax);
342 }
343 }
344
ExitRegionIndirect(Register target)345 void LiteTranslator::ExitRegionIndirect(Register target) {
346 StoreMappedRegs();
347 if (params_.allow_dispatch) {
348 EmitIndirectDispatch(&as_, target);
349 } else {
350 EmitExitGeneratedCode(&as_, target);
351 }
352 }
353
Branch(int32_t offset)354 void LiteTranslator::Branch(int32_t offset) {
355 is_region_end_reached_ = true;
356 ExitRegion(GetInsnAddr() + offset);
357 }
358
BranchRegister(Register base,int16_t offset)359 void LiteTranslator::BranchRegister(Register base, int16_t offset) {
360 Register res = AllocTempReg();
361 as_.Movq(res, base);
362 as_.Addq(res, offset);
363 // TODO(b/232598137) Maybe move this to translation cache?
364 // Zeroing out the last bit.
365 as_.Andq(res, ~int32_t{1});
366 is_region_end_reached_ = true;
367 ExitRegionIndirect(res);
368 }
369
Load(Decoder::LoadOperandType operand_type,Register arg,int16_t offset)370 Register LiteTranslator::Load(Decoder::LoadOperandType operand_type, Register arg, int16_t offset) {
371 AssemblerBase::Label* recovery_label = as_.MakeLabel();
372 as_.SetRecoveryPoint(recovery_label);
373
374 Register res = AllocTempReg();
375 Assembler::Operand asm_memop{.base = arg, .disp = offset};
376 switch (operand_type) {
377 case Decoder::LoadOperandType::k8bitUnsigned:
378 as_.Movzxbl(res, asm_memop);
379 break;
380 case Decoder::LoadOperandType::k16bitUnsigned:
381 as_.Movzxwl(res, asm_memop);
382 break;
383 case Decoder::LoadOperandType::k32bitUnsigned:
384 as_.Movl(res, asm_memop);
385 break;
386 case Decoder::LoadOperandType::k64bit:
387 as_.Movq(res, asm_memop);
388 break;
389 case Decoder::LoadOperandType::k8bitSigned:
390 as_.Movsxbq(res, asm_memop);
391 break;
392 case Decoder::LoadOperandType::k16bitSigned:
393 as_.Movsxwq(res, asm_memop);
394 break;
395 case Decoder::LoadOperandType::k32bitSigned:
396 as_.Movsxlq(res, asm_memop);
397 break;
398 default:
399 Undefined();
400 return {};
401 }
402
403 // TODO(b/144326673): Emit the recovery code at the end of the region so it doesn't interrupt
404 // normal code flow with the jump and doesn't negatively affect instruction cache locality.
405 AssemblerBase::Label* cont = as_.MakeLabel();
406 as_.Jmp(*cont);
407 as_.Bind(recovery_label);
408 ExitGeneratedCode(GetInsnAddr());
409 as_.Bind(cont);
410
411 return res;
412 }
413
Store(Decoder::MemoryDataOperandType operand_type,Register arg,int16_t offset,Register data)414 void LiteTranslator::Store(Decoder::MemoryDataOperandType operand_type,
415 Register arg,
416 int16_t offset,
417 Register data) {
418 AssemblerBase::Label* recovery_label = as_.MakeLabel();
419 as_.SetRecoveryPoint(recovery_label);
420
421 Assembler::Operand asm_memop{.base = arg, .disp = offset};
422 switch (operand_type) {
423 case Decoder::MemoryDataOperandType::k8bit:
424 as_.Movb(asm_memop, data);
425 break;
426 case Decoder::MemoryDataOperandType::k16bit:
427 as_.Movw(asm_memop, data);
428 break;
429 case Decoder::MemoryDataOperandType::k32bit:
430 as_.Movl(asm_memop, data);
431 break;
432 case Decoder::MemoryDataOperandType::k64bit:
433 as_.Movq(asm_memop, data);
434 break;
435 default:
436 return Undefined();
437 }
438
439 // TODO(b/144326673): Emit the recovery code at the end of the region so it doesn't interrupt
440 // normal code flow with the jump and doesn't negatively affect instruction cache locality.
441 AssemblerBase::Label* cont = as_.MakeLabel();
442 as_.Jmp(*cont);
443 as_.Bind(recovery_label);
444 ExitGeneratedCode(GetInsnAddr());
445 as_.Bind(cont);
446 }
447
UpdateCsr(Decoder::CsrOpcode opcode,Register arg,Register csr)448 Register LiteTranslator::UpdateCsr(Decoder::CsrOpcode opcode, Register arg, Register csr) {
449 Register res = AllocTempReg();
450 switch (opcode) {
451 case Decoder::CsrOpcode::kCsrrs:
452 as_.Movq(res, arg);
453 as_.Orq(res, csr);
454 break;
455 case Decoder::CsrOpcode::kCsrrc:
456 if (host_platform::kHasBMI) {
457 as_.Andnq(res, arg, csr);
458 } else {
459 as_.Movq(res, arg);
460 as_.Notq(res);
461 as_.Andq(res, csr);
462 }
463 break;
464 default:
465 Undefined();
466 return {};
467 }
468 return res;
469 }
470
UpdateCsr(Decoder::CsrImmOpcode opcode,uint8_t imm,Register csr)471 Register LiteTranslator::UpdateCsr(Decoder::CsrImmOpcode opcode, uint8_t imm, Register csr) {
472 Register res = AllocTempReg();
473 switch (opcode) {
474 case Decoder::CsrImmOpcode::kCsrrwi:
475 as_.Movl(res, imm);
476 break;
477 case Decoder::CsrImmOpcode::kCsrrsi:
478 as_.Movl(res, imm);
479 as_.Orq(res, csr);
480 break;
481 case Decoder::CsrImmOpcode::kCsrrci:
482 as_.Movq(res, static_cast<int8_t>(~imm));
483 as_.Andq(res, csr);
484 break;
485 default:
486 Undefined();
487 return {};
488 }
489 return res;
490 }
491
492 } // namespace berberis
493