1 /*
2  * Copyright (C) 2016 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include "jni_macro_assembler_arm64.h"
18 
19 #include "entrypoints/quick/quick_entrypoints.h"
20 #include "indirect_reference_table.h"
21 #include "jni/jni_env_ext.h"
22 #include "jni/local_reference_table.h"
23 #include "lock_word.h"
24 #include "managed_register_arm64.h"
25 #include "offsets.h"
26 #include "thread.h"
27 
28 using namespace vixl::aarch64;  // NOLINT(build/namespaces)
29 
30 namespace art HIDDEN {
31 namespace arm64 {
32 
33 #ifdef ___
34 #error "ARM64 Assembler macro already defined."
35 #else
36 #define ___   asm_.GetVIXLAssembler()->
37 #endif
38 
39 #define reg_x(X) Arm64Assembler::reg_x(X)
40 #define reg_w(W) Arm64Assembler::reg_w(W)
41 #define reg_d(D) Arm64Assembler::reg_d(D)
42 #define reg_s(S) Arm64Assembler::reg_s(S)
43 
44 // The AAPCS64 requires 16-byte alignment. This is the same as the Managed ABI stack alignment.
45 static constexpr size_t kAapcs64StackAlignment = 16u;
46 static_assert(kAapcs64StackAlignment == kStackAlignment);
47 
48 // STP signed offset for W-register can encode any 4-byte aligned offset smaller than this cutoff.
49 static constexpr size_t kStpWOffsetCutoff = 256u;
50 
51 // STP signed offset for X-register can encode any 8-byte aligned offset smaller than this cutoff.
52 static constexpr size_t kStpXOffsetCutoff = 512u;
53 
54 // STP signed offset for S-register can encode any 4-byte aligned offset smaller than this cutoff.
55 static constexpr size_t kStpSOffsetCutoff = 256u;
56 
57 // STP signed offset for D-register can encode any 8-byte aligned offset smaller than this cutoff.
58 static constexpr size_t kStpDOffsetCutoff = 512u;
59 
~Arm64JNIMacroAssembler()60 Arm64JNIMacroAssembler::~Arm64JNIMacroAssembler() {
61 }
62 
FinalizeCode()63 void Arm64JNIMacroAssembler::FinalizeCode() {
64   ___ FinalizeCode();
65 }
66 
GetCurrentThread(ManagedRegister dest)67 void Arm64JNIMacroAssembler::GetCurrentThread(ManagedRegister dest) {
68   ___ Mov(reg_x(dest.AsArm64().AsXRegister()), reg_x(TR));
69 }
70 
GetCurrentThread(FrameOffset offset)71 void Arm64JNIMacroAssembler::GetCurrentThread(FrameOffset offset) {
72   StoreToOffset(TR, SP, offset.Int32Value());
73 }
74 
75 // See Arm64 PCS Section 5.2.2.1.
IncreaseFrameSize(size_t adjust)76 void Arm64JNIMacroAssembler::IncreaseFrameSize(size_t adjust) {
77   if (adjust != 0u) {
78     CHECK_ALIGNED(adjust, kStackAlignment);
79     AddConstant(SP, -adjust);
80     cfi().AdjustCFAOffset(adjust);
81   }
82 }
83 
84 // See Arm64 PCS Section 5.2.2.1.
DecreaseFrameSize(size_t adjust)85 void Arm64JNIMacroAssembler::DecreaseFrameSize(size_t adjust) {
86   if (adjust != 0u) {
87     CHECK_ALIGNED(adjust, kStackAlignment);
88     AddConstant(SP, adjust);
89     cfi().AdjustCFAOffset(-adjust);
90   }
91 }
92 
CoreRegisterWithSize(ManagedRegister m_src,size_t size)93 ManagedRegister Arm64JNIMacroAssembler::CoreRegisterWithSize(ManagedRegister m_src, size_t size) {
94   DCHECK(size == 4u || size == 8u) << size;
95   Arm64ManagedRegister src = m_src.AsArm64();
96   // Switch between X and W registers using the `XRegister` and `WRegister` enumerations.
97   static_assert(W0 == static_cast<std::underlying_type_t<XRegister>>(X0));
98   static_assert(W30 == static_cast<std::underlying_type_t<XRegister>>(X30));
99   static_assert(WSP == static_cast<std::underlying_type_t<XRegister>>(SP));
100   static_assert(WZR == static_cast<std::underlying_type_t<XRegister>>(XZR));
101   if (src.IsXRegister()) {
102     if (size == 8u) {
103       return m_src;
104     }
105     auto id = static_cast<std::underlying_type_t<XRegister>>(src.AsXRegister());
106     return Arm64ManagedRegister::FromWRegister(enum_cast<WRegister>(id));
107   } else {
108     CHECK(src.IsWRegister());
109     if (size == 4u) {
110       return m_src;
111     }
112     auto id = static_cast<std::underlying_type_t<WRegister>>(src.AsWRegister());
113     return Arm64ManagedRegister::FromXRegister(enum_cast<XRegister>(id));
114   }
115 }
116 
AddConstant(XRegister rd,int32_t value,Condition cond)117 void Arm64JNIMacroAssembler::AddConstant(XRegister rd, int32_t value, Condition cond) {
118   AddConstant(rd, rd, value, cond);
119 }
120 
AddConstant(XRegister rd,XRegister rn,int32_t value,Condition cond)121 void Arm64JNIMacroAssembler::AddConstant(XRegister rd,
122                                          XRegister rn,
123                                          int32_t value,
124                                          Condition cond) {
125   if ((cond == al) || (cond == nv)) {
126     // VIXL macro-assembler handles all variants.
127     ___ Add(reg_x(rd), reg_x(rn), value);
128   } else {
129     // temp = rd + value
130     // rd = cond ? temp : rn
131     UseScratchRegisterScope temps(asm_.GetVIXLAssembler());
132     temps.Exclude(reg_x(rd), reg_x(rn));
133     Register temp = temps.AcquireX();
134     ___ Add(temp, reg_x(rn), value);
135     ___ Csel(reg_x(rd), temp, reg_x(rd), cond);
136   }
137 }
138 
StoreWToOffset(StoreOperandType type,WRegister source,XRegister base,int32_t offset)139 void Arm64JNIMacroAssembler::StoreWToOffset(StoreOperandType type,
140                                             WRegister source,
141                                             XRegister base,
142                                             int32_t offset) {
143   switch (type) {
144     case kStoreByte:
145       ___ Strb(reg_w(source), MEM_OP(reg_x(base), offset));
146       break;
147     case kStoreHalfword:
148       ___ Strh(reg_w(source), MEM_OP(reg_x(base), offset));
149       break;
150     case kStoreWord:
151       ___ Str(reg_w(source), MEM_OP(reg_x(base), offset));
152       break;
153     default:
154       LOG(FATAL) << "UNREACHABLE";
155   }
156 }
157 
StoreToOffset(XRegister source,XRegister base,int32_t offset)158 void Arm64JNIMacroAssembler::StoreToOffset(XRegister source, XRegister base, int32_t offset) {
159   CHECK_NE(source, SP);
160   ___ Str(reg_x(source), MEM_OP(reg_x(base), offset));
161 }
162 
StoreSToOffset(SRegister source,XRegister base,int32_t offset)163 void Arm64JNIMacroAssembler::StoreSToOffset(SRegister source, XRegister base, int32_t offset) {
164   ___ Str(reg_s(source), MEM_OP(reg_x(base), offset));
165 }
166 
StoreDToOffset(DRegister source,XRegister base,int32_t offset)167 void Arm64JNIMacroAssembler::StoreDToOffset(DRegister source, XRegister base, int32_t offset) {
168   ___ Str(reg_d(source), MEM_OP(reg_x(base), offset));
169 }
170 
Store(FrameOffset offs,ManagedRegister m_src,size_t size)171 void Arm64JNIMacroAssembler::Store(FrameOffset offs, ManagedRegister m_src, size_t size) {
172   Store(Arm64ManagedRegister::FromXRegister(SP), MemberOffset(offs.Int32Value()), m_src, size);
173 }
174 
Store(ManagedRegister m_base,MemberOffset offs,ManagedRegister m_src,size_t size)175 void Arm64JNIMacroAssembler::Store(ManagedRegister m_base,
176                                    MemberOffset offs,
177                                    ManagedRegister m_src,
178                                    size_t size) {
179   Arm64ManagedRegister base = m_base.AsArm64();
180   Arm64ManagedRegister src = m_src.AsArm64();
181   if (src.IsNoRegister()) {
182     CHECK_EQ(0u, size);
183   } else if (src.IsWRegister()) {
184     CHECK_EQ(4u, size);
185     StoreWToOffset(kStoreWord, src.AsWRegister(), base.AsXRegister(), offs.Int32Value());
186   } else if (src.IsXRegister()) {
187     CHECK_EQ(8u, size);
188     StoreToOffset(src.AsXRegister(), base.AsXRegister(), offs.Int32Value());
189   } else if (src.IsSRegister()) {
190     StoreSToOffset(src.AsSRegister(), base.AsXRegister(), offs.Int32Value());
191   } else {
192     CHECK(src.IsDRegister()) << src;
193     StoreDToOffset(src.AsDRegister(), base.AsXRegister(), offs.Int32Value());
194   }
195 }
196 
StoreRawPtr(FrameOffset offs,ManagedRegister m_src)197 void Arm64JNIMacroAssembler::StoreRawPtr(FrameOffset offs, ManagedRegister m_src) {
198   Arm64ManagedRegister src = m_src.AsArm64();
199   CHECK(src.IsXRegister()) << src;
200   StoreToOffset(src.AsXRegister(), SP, offs.Int32Value());
201 }
202 
StoreStackPointerToThread(ThreadOffset64 tr_offs,bool tag_sp)203 void Arm64JNIMacroAssembler::StoreStackPointerToThread(ThreadOffset64 tr_offs, bool tag_sp) {
204   UseScratchRegisterScope temps(asm_.GetVIXLAssembler());
205   Register scratch = temps.AcquireX();
206   ___ Mov(scratch, reg_x(SP));
207   if (tag_sp) {
208     ___ Orr(scratch, scratch, 0x2);
209   }
210   ___ Str(scratch, MEM_OP(reg_x(TR), tr_offs.Int32Value()));
211 }
212 
213 // Load routines.
LoadImmediate(XRegister dest,int32_t value,Condition cond)214 void Arm64JNIMacroAssembler::LoadImmediate(XRegister dest, int32_t value, Condition cond) {
215   if ((cond == al) || (cond == nv)) {
216     ___ Mov(reg_x(dest), value);
217   } else {
218     // temp = value
219     // rd = cond ? temp : rd
220     if (value != 0) {
221       UseScratchRegisterScope temps(asm_.GetVIXLAssembler());
222       temps.Exclude(reg_x(dest));
223       Register temp = temps.AcquireX();
224       ___ Mov(temp, value);
225       ___ Csel(reg_x(dest), temp, reg_x(dest), cond);
226     } else {
227       ___ Csel(reg_x(dest), reg_x(XZR), reg_x(dest), cond);
228     }
229   }
230 }
231 
LoadWFromOffset(LoadOperandType type,WRegister dest,XRegister base,int32_t offset)232 void Arm64JNIMacroAssembler::LoadWFromOffset(LoadOperandType type,
233                                              WRegister dest,
234                                              XRegister base,
235                                              int32_t offset) {
236   switch (type) {
237     case kLoadSignedByte:
238       ___ Ldrsb(reg_w(dest), MEM_OP(reg_x(base), offset));
239       break;
240     case kLoadSignedHalfword:
241       ___ Ldrsh(reg_w(dest), MEM_OP(reg_x(base), offset));
242       break;
243     case kLoadUnsignedByte:
244       ___ Ldrb(reg_w(dest), MEM_OP(reg_x(base), offset));
245       break;
246     case kLoadUnsignedHalfword:
247       ___ Ldrh(reg_w(dest), MEM_OP(reg_x(base), offset));
248       break;
249     case kLoadWord:
250       ___ Ldr(reg_w(dest), MEM_OP(reg_x(base), offset));
251       break;
252     default:
253         LOG(FATAL) << "UNREACHABLE";
254   }
255 }
256 
257 // Note: We can extend this member by adding load type info - see
258 // sign extended A64 load variants.
LoadFromOffset(XRegister dest,XRegister base,int32_t offset)259 void Arm64JNIMacroAssembler::LoadFromOffset(XRegister dest, XRegister base, int32_t offset) {
260   CHECK_NE(dest, SP);
261   ___ Ldr(reg_x(dest), MEM_OP(reg_x(base), offset));
262 }
263 
LoadSFromOffset(SRegister dest,XRegister base,int32_t offset)264 void Arm64JNIMacroAssembler::LoadSFromOffset(SRegister dest, XRegister base, int32_t offset) {
265   ___ Ldr(reg_s(dest), MEM_OP(reg_x(base), offset));
266 }
267 
LoadDFromOffset(DRegister dest,XRegister base,int32_t offset)268 void Arm64JNIMacroAssembler::LoadDFromOffset(DRegister dest, XRegister base, int32_t offset) {
269   ___ Ldr(reg_d(dest), MEM_OP(reg_x(base), offset));
270 }
271 
Load(Arm64ManagedRegister dest,XRegister base,int32_t offset,size_t size)272 void Arm64JNIMacroAssembler::Load(Arm64ManagedRegister dest,
273                                   XRegister base,
274                                   int32_t offset,
275                                   size_t size) {
276   if (dest.IsNoRegister()) {
277     CHECK_EQ(0u, size) << dest;
278   } else if (dest.IsWRegister()) {
279     CHECK_EQ(4u, size) << dest;
280     ___ Ldr(reg_w(dest.AsWRegister()), MEM_OP(reg_x(base), offset));
281   } else if (dest.IsXRegister()) {
282     CHECK_NE(dest.AsXRegister(), SP) << dest;
283 
284     if (size == 1u) {
285       ___ Ldrb(reg_w(dest.AsOverlappingWRegister()), MEM_OP(reg_x(base), offset));
286     } else if (size == 4u) {
287       ___ Ldr(reg_w(dest.AsOverlappingWRegister()), MEM_OP(reg_x(base), offset));
288     }  else {
289       CHECK_EQ(8u, size) << dest;
290       ___ Ldr(reg_x(dest.AsXRegister()), MEM_OP(reg_x(base), offset));
291     }
292   } else if (dest.IsSRegister()) {
293     ___ Ldr(reg_s(dest.AsSRegister()), MEM_OP(reg_x(base), offset));
294   } else {
295     CHECK(dest.IsDRegister()) << dest;
296     ___ Ldr(reg_d(dest.AsDRegister()), MEM_OP(reg_x(base), offset));
297   }
298 }
299 
Load(ManagedRegister m_dst,FrameOffset src,size_t size)300 void Arm64JNIMacroAssembler::Load(ManagedRegister m_dst, FrameOffset src, size_t size) {
301   return Load(m_dst.AsArm64(), SP, src.Int32Value(), size);
302 }
303 
Load(ManagedRegister m_dst,ManagedRegister m_base,MemberOffset offs,size_t size)304 void Arm64JNIMacroAssembler::Load(ManagedRegister m_dst,
305                                   ManagedRegister m_base,
306                                   MemberOffset offs,
307                                   size_t size) {
308   return Load(m_dst.AsArm64(), m_base.AsArm64().AsXRegister(), offs.Int32Value(), size);
309 }
310 
LoadRawPtrFromThread(ManagedRegister m_dst,ThreadOffset64 offs)311 void Arm64JNIMacroAssembler::LoadRawPtrFromThread(ManagedRegister m_dst, ThreadOffset64 offs) {
312   Arm64ManagedRegister dst = m_dst.AsArm64();
313   CHECK(dst.IsXRegister()) << dst;
314   LoadFromOffset(dst.AsXRegister(), TR, offs.Int32Value());
315 }
316 
317 // Copying routines.
MoveArguments(ArrayRef<ArgumentLocation> dests,ArrayRef<ArgumentLocation> srcs,ArrayRef<FrameOffset> refs)318 void Arm64JNIMacroAssembler::MoveArguments(ArrayRef<ArgumentLocation> dests,
319                                            ArrayRef<ArgumentLocation> srcs,
320                                            ArrayRef<FrameOffset> refs) {
321   size_t arg_count = dests.size();
322   DCHECK_EQ(arg_count, srcs.size());
323   DCHECK_EQ(arg_count, refs.size());
324 
325   auto get_mask = [](ManagedRegister reg) -> uint64_t {
326     Arm64ManagedRegister arm64_reg = reg.AsArm64();
327     if (arm64_reg.IsXRegister()) {
328       size_t core_reg_number = static_cast<size_t>(arm64_reg.AsXRegister());
329       DCHECK_LT(core_reg_number, 31u);  // xSP, xZR not allowed.
330       return UINT64_C(1) << core_reg_number;
331     } else if (arm64_reg.IsWRegister()) {
332       size_t core_reg_number = static_cast<size_t>(arm64_reg.AsWRegister());
333       DCHECK_LT(core_reg_number, 31u);  // wSP, wZR not allowed.
334       return UINT64_C(1) << core_reg_number;
335     } else if (arm64_reg.IsDRegister()) {
336       size_t fp_reg_number = static_cast<size_t>(arm64_reg.AsDRegister());
337       DCHECK_LT(fp_reg_number, 32u);
338       return (UINT64_C(1) << 32u) << fp_reg_number;
339     } else {
340       DCHECK(arm64_reg.IsSRegister());
341       size_t fp_reg_number = static_cast<size_t>(arm64_reg.AsSRegister());
342       DCHECK_LT(fp_reg_number, 32u);
343       return (UINT64_C(1) << 32u) << fp_reg_number;
344     }
345   };
346 
347   // More than 8 core or FP reg args are very rare, so we do not optimize for
348   // that case by using LDP/STP, except for situations that arise even with low
349   // number of arguments. We use STP for the non-reference spilling which also
350   // covers the initial spill for native reference register args as they are
351   // spilled as raw 32-bit values. We also optimize loading args to registers
352   // with LDP, whether references or not, except for the initial non-null
353   // reference which we do not need to load at all.
354 
355   // Collect registers to move while storing/copying args to stack slots.
356   // Convert processed references to `jobject`.
357   uint64_t src_regs = 0u;
358   uint64_t dest_regs = 0u;
359   for (size_t i = 0; i != arg_count; ++i) {
360     const ArgumentLocation& src = srcs[i];
361     const ArgumentLocation& dest = dests[i];
362     const FrameOffset ref = refs[i];
363     if (ref != kInvalidReferenceOffset) {
364       DCHECK_EQ(src.GetSize(), kObjectReferenceSize);
365       DCHECK_EQ(dest.GetSize(), static_cast<size_t>(kArm64PointerSize));
366     } else {
367       DCHECK_EQ(src.GetSize(), dest.GetSize());
368     }
369     if (dest.IsRegister()) {
370       // Note: For references, `Equals()` returns `false` for overlapping W and X registers.
371       if (ref != kInvalidReferenceOffset &&
372           src.IsRegister() &&
373           src.GetRegister().AsArm64().AsOverlappingXRegister() ==
374               dest.GetRegister().AsArm64().AsXRegister()) {
375         // Just convert to `jobject`. No further processing is needed.
376         CreateJObject(dest.GetRegister(), ref, src.GetRegister(), /*null_allowed=*/ i != 0u);
377       } else if (src.IsRegister() && src.GetRegister().Equals(dest.GetRegister())) {
378         // Nothing to do.
379       } else {
380         if (src.IsRegister()) {
381           src_regs |= get_mask(src.GetRegister());
382         }
383         dest_regs |= get_mask(dest.GetRegister());
384       }
385     } else if (ref != kInvalidReferenceOffset) {
386       if (src.IsRegister()) {
387         // Note: We can clobber `src` here as the register cannot hold more than one argument.
388         ManagedRegister src_x =
389             CoreRegisterWithSize(src.GetRegister(), static_cast<size_t>(kArm64PointerSize));
390         CreateJObject(src_x, ref, src.GetRegister(), /*null_allowed=*/ i != 0u);
391         Store(dest.GetFrameOffset(), src_x, dest.GetSize());
392       } else {
393         CreateJObject(dest.GetFrameOffset(), ref, /*null_allowed=*/ i != 0u);
394       }
395     } else {
396       if (src.IsRegister()) {
397         static_assert(kStpWOffsetCutoff == kStpSOffsetCutoff);
398         static_assert(kStpXOffsetCutoff == kStpDOffsetCutoff);
399         if (i + 1u != arg_count &&
400             srcs[i + 1u].IsRegister() &&
401             srcs[i + 1u].GetSize() == dest.GetSize() &&
402             src.GetRegister().AsArm64().IsGPRegister() ==
403                 srcs[i + 1u].GetRegister().AsArm64().IsGPRegister() &&
404             refs[i + 1u] == kInvalidReferenceOffset &&
405             !dests[i + 1u].IsRegister() &&
406             dests[i + 1u].GetFrameOffset().SizeValue() ==
407                 dest.GetFrameOffset().SizeValue() + dest.GetSize() &&
408             IsAlignedParam(dest.GetFrameOffset().SizeValue(), dest.GetSize()) &&
409             dest.GetFrameOffset().SizeValue() <
410                 (dest.GetSize() == 8u ? kStpXOffsetCutoff : kStpWOffsetCutoff)) {
411           DCHECK_EQ(dests[i + 1u].GetSize(), dest.GetSize());
412           Arm64ManagedRegister src_reg = src.GetRegister().AsArm64();
413           Arm64ManagedRegister src2_reg = srcs[i + 1u].GetRegister().AsArm64();
414           DCHECK_EQ(dest.GetSize() == 8u, src_reg.IsXRegister() || src_reg.IsDRegister());
415           DCHECK_EQ(dest.GetSize() == 8u, src2_reg.IsXRegister() || src2_reg.IsDRegister());
416           if (src_reg.IsWRegister()) {
417             ___ Stp(reg_w(src_reg.AsWRegister()),
418                     reg_w(src2_reg.AsWRegister()),
419                     MEM_OP(sp, dest.GetFrameOffset().SizeValue()));
420           } else if (src_reg.IsXRegister()) {
421             ___ Stp(reg_x(src_reg.AsXRegister()),
422                     reg_x(src2_reg.AsXRegister()),
423                     MEM_OP(sp, dest.GetFrameOffset().SizeValue()));
424           } else if (src_reg.IsSRegister()) {
425             ___ Stp(reg_s(src_reg.AsSRegister()),
426                     reg_s(src2_reg.AsSRegister()),
427                     MEM_OP(sp, dest.GetFrameOffset().SizeValue()));
428           } else {
429             DCHECK(src_reg.IsDRegister());
430             ___ Stp(reg_d(src_reg.AsDRegister()),
431                     reg_d(src2_reg.AsDRegister()),
432                     MEM_OP(sp, dest.GetFrameOffset().SizeValue()));
433           }
434           ++i;
435         } else {
436           Store(dest.GetFrameOffset(), src.GetRegister(), dest.GetSize());
437         }
438       } else {
439         Copy(dest.GetFrameOffset(), src.GetFrameOffset(), dest.GetSize());
440       }
441     }
442   }
443   // Fill destination registers.
444   // There should be no cycles, so this simple algorithm should make progress.
445   while (dest_regs != 0u) {
446     uint64_t old_dest_regs = dest_regs;
447     for (size_t i = 0; i != arg_count; ++i) {
448       const ArgumentLocation& src = srcs[i];
449       const ArgumentLocation& dest = dests[i];
450       const FrameOffset ref = refs[i];
451       if (!dest.IsRegister()) {
452         continue;  // Stored in first loop above.
453       }
454       auto can_process = [&](ManagedRegister dest_reg) {
455         uint64_t dest_reg_mask = get_mask(dest_reg);
456         if ((dest_reg_mask & dest_regs) == 0u) {
457           return false;  // Equals source, or already filled in one of previous iterations.
458         }
459         if ((dest_reg_mask & src_regs) != 0u) {
460           return false;  // Cannot clobber this register yet.
461         }
462         return true;
463       };
464       if (!can_process(dest.GetRegister())) {
465         continue;
466       }
467       if (src.IsRegister()) {
468         if (ref != kInvalidReferenceOffset) {
469           CreateJObject(dest.GetRegister(), ref, src.GetRegister(), /*null_allowed=*/ i != 0u);
470         } else {
471           Move(dest.GetRegister(), src.GetRegister(), dest.GetSize());
472         }
473         src_regs &= ~get_mask(src.GetRegister());  // Allow clobbering source register.
474       } else if (i + 1u != arg_count &&
475                  (i != 0u || ref == kInvalidReferenceOffset) &&  // Not for non-null reference.
476                  dests[i + 1u].IsRegister() &&
477                  dest.GetRegister().AsArm64().IsGPRegister() ==
478                      dests[i + 1u].GetRegister().AsArm64().IsGPRegister() &&
479                  !srcs[i + 1u].IsRegister() &&
480                  srcs[i + 1u].GetSize() == src.GetSize() &&
481                  srcs[i + 1u].GetFrameOffset().SizeValue() ==
482                      src.GetFrameOffset().SizeValue() + src.GetSize() &&
483                  IsAlignedParam(src.GetFrameOffset().SizeValue(), src.GetSize()) &&
484                  can_process(dests[i + 1u].GetRegister())) {
485         Arm64ManagedRegister dest_reg = dest.GetRegister().AsArm64();
486         Arm64ManagedRegister dest2_reg = dests[i + 1u].GetRegister().AsArm64();
487         DCHECK(ref == kInvalidReferenceOffset || dest_reg.IsXRegister());
488         DCHECK(refs[i + 1u] == kInvalidReferenceOffset || dest2_reg.IsXRegister());
489         if (dest_reg.IsDRegister()) {
490           DCHECK_EQ(dest.GetSize(), 8u);
491           DCHECK_EQ(dests[i + 1u].GetSize(), 8u);
492           ___ Ldp(reg_d(dest_reg.AsDRegister()),
493                   reg_d(dest2_reg.AsDRegister()),
494                   MEM_OP(sp, src.GetFrameOffset().SizeValue()));
495         } else if (dest_reg.IsSRegister()) {
496           DCHECK_EQ(dest.GetSize(), 4u);
497           DCHECK_EQ(dests[i + 1u].GetSize(), 4u);
498           ___ Ldp(reg_s(dest_reg.AsSRegister()),
499                   reg_s(dest2_reg.AsSRegister()),
500                   MEM_OP(sp, src.GetFrameOffset().SizeValue()));
501         } else if (src.GetSize() == 8u) {
502           DCHECK_EQ(dest.GetSize(), 8u);
503           DCHECK_EQ(dests[i + 1u].GetSize(), 8u);
504           ___ Ldp(reg_x(dest_reg.AsXRegister()),
505                   reg_x(dest2_reg.AsXRegister()),
506                   MEM_OP(sp, src.GetFrameOffset().SizeValue()));
507         } else {
508           DCHECK_EQ(dest.GetSize(), ref != kInvalidReferenceOffset ? 8u : 4u);
509           DCHECK_EQ(dests[i + 1u].GetSize(), refs[i + 1u] != kInvalidReferenceOffset ? 8u : 4u);
510           auto to_w = [](Arm64ManagedRegister reg) {
511             return reg_w(reg.IsXRegister() ? reg.AsOverlappingWRegister() : reg.AsWRegister());
512           };
513           ___ Ldp(to_w(dest_reg), to_w(dest2_reg), MEM_OP(sp, src.GetFrameOffset().SizeValue()));
514           auto to_mr_w = [](Arm64ManagedRegister reg) {
515             return Arm64ManagedRegister::FromWRegister(reg.AsOverlappingWRegister());
516           };
517           if (ref != kInvalidReferenceOffset) {
518             CreateJObject(dest_reg, ref, to_mr_w(dest_reg), /*null_allowed=*/ true);
519           }
520           if (refs[i + 1u] != kInvalidReferenceOffset) {
521             CreateJObject(dest2_reg, refs[i + 1u], to_mr_w(dest2_reg), /*null_allowed=*/ true);
522           }
523         }
524         dest_regs &= ~get_mask(dest2_reg);  // Destination register was filled.
525         ++i;  // Proceed to mark the other destination register as filled.
526       } else {
527         if (ref != kInvalidReferenceOffset) {
528           CreateJObject(
529               dest.GetRegister(), ref, ManagedRegister::NoRegister(), /*null_allowed=*/ i != 0u);
530         } else {
531           Load(dest.GetRegister(), src.GetFrameOffset(), dest.GetSize());
532         }
533       }
534       dest_regs &= ~get_mask(dest.GetRegister());  // Destination register was filled.
535     }
536     CHECK_NE(old_dest_regs, dest_regs);
537     DCHECK_EQ(0u, dest_regs & ~old_dest_regs);
538   }
539 }
540 
Move(ManagedRegister m_dst,ManagedRegister m_src,size_t size)541 void Arm64JNIMacroAssembler::Move(ManagedRegister m_dst, ManagedRegister m_src, size_t size) {
542   Arm64ManagedRegister dst = m_dst.AsArm64();
543   if (kIsDebugBuild) {
544     // Check that the destination is not a scratch register.
545     UseScratchRegisterScope temps(asm_.GetVIXLAssembler());
546     if (dst.IsXRegister()) {
547       CHECK(!temps.IsAvailable(reg_x(dst.AsXRegister())));
548     } else if (dst.IsWRegister()) {
549       CHECK(!temps.IsAvailable(reg_w(dst.AsWRegister())));
550     } else if (dst.IsSRegister()) {
551       CHECK(!temps.IsAvailable(reg_s(dst.AsSRegister())));
552     } else {
553       CHECK(!temps.IsAvailable(reg_d(dst.AsDRegister())));
554     }
555   }
556   Arm64ManagedRegister src = m_src.AsArm64();
557   if (!dst.Equals(src)) {
558     if (dst.IsXRegister()) {
559       if (size == 4) {
560         CHECK(src.IsWRegister());
561         ___ Mov(reg_w(dst.AsOverlappingWRegister()), reg_w(src.AsWRegister()));
562       } else {
563         if (src.IsXRegister()) {
564           ___ Mov(reg_x(dst.AsXRegister()), reg_x(src.AsXRegister()));
565         } else {
566           ___ Mov(reg_x(dst.AsXRegister()), reg_x(src.AsOverlappingXRegister()));
567         }
568       }
569     } else if (dst.IsWRegister()) {
570       CHECK(src.IsWRegister()) << src;
571       ___ Mov(reg_w(dst.AsWRegister()), reg_w(src.AsWRegister()));
572     } else if (dst.IsSRegister()) {
573       CHECK(src.IsSRegister()) << src;
574       ___ Fmov(reg_s(dst.AsSRegister()), reg_s(src.AsSRegister()));
575     } else {
576       CHECK(dst.IsDRegister()) << dst;
577       CHECK(src.IsDRegister()) << src;
578       ___ Fmov(reg_d(dst.AsDRegister()), reg_d(src.AsDRegister()));
579     }
580   }
581 }
582 
Move(ManagedRegister m_dst,size_t value)583 void Arm64JNIMacroAssembler::Move(ManagedRegister m_dst, size_t value) {
584   Arm64ManagedRegister dst = m_dst.AsArm64();
585   DCHECK(dst.IsXRegister());
586   ___ Mov(reg_x(dst.AsXRegister()), value);
587 }
588 
Copy(FrameOffset dest,FrameOffset src,size_t size)589 void Arm64JNIMacroAssembler::Copy(FrameOffset dest, FrameOffset src, size_t size) {
590   DCHECK(size == 4 || size == 8) << size;
591   UseScratchRegisterScope temps(asm_.GetVIXLAssembler());
592   Register scratch = (size == 8) ? temps.AcquireX() : temps.AcquireW();
593   ___ Ldr(scratch, MEM_OP(reg_x(SP), src.Int32Value()));
594   ___ Str(scratch, MEM_OP(reg_x(SP), dest.Int32Value()));
595 }
596 
SignExtend(ManagedRegister mreg,size_t size)597 void Arm64JNIMacroAssembler::SignExtend(ManagedRegister mreg, size_t size) {
598   Arm64ManagedRegister reg = mreg.AsArm64();
599   CHECK(size == 1 || size == 2) << size;
600   CHECK(reg.IsWRegister()) << reg;
601   if (size == 1) {
602     ___ Sxtb(reg_w(reg.AsWRegister()), reg_w(reg.AsWRegister()));
603   } else {
604     ___ Sxth(reg_w(reg.AsWRegister()), reg_w(reg.AsWRegister()));
605   }
606 }
607 
ZeroExtend(ManagedRegister mreg,size_t size)608 void Arm64JNIMacroAssembler::ZeroExtend(ManagedRegister mreg, size_t size) {
609   Arm64ManagedRegister reg = mreg.AsArm64();
610   CHECK(size == 1 || size == 2) << size;
611   CHECK(reg.IsWRegister()) << reg;
612   if (size == 1) {
613     ___ Uxtb(reg_w(reg.AsWRegister()), reg_w(reg.AsWRegister()));
614   } else {
615     ___ Uxth(reg_w(reg.AsWRegister()), reg_w(reg.AsWRegister()));
616   }
617 }
618 
VerifyObject(ManagedRegister,bool)619 void Arm64JNIMacroAssembler::VerifyObject(ManagedRegister /*src*/, bool /*could_be_null*/) {
620   // TODO: not validating references.
621 }
622 
VerifyObject(FrameOffset,bool)623 void Arm64JNIMacroAssembler::VerifyObject(FrameOffset /*src*/, bool /*could_be_null*/) {
624   // TODO: not validating references.
625 }
626 
Jump(ManagedRegister m_base,Offset offs)627 void Arm64JNIMacroAssembler::Jump(ManagedRegister m_base, Offset offs) {
628   Arm64ManagedRegister base = m_base.AsArm64();
629   CHECK(base.IsXRegister()) << base;
630   UseScratchRegisterScope temps(asm_.GetVIXLAssembler());
631   Register scratch = temps.AcquireX();
632   ___ Ldr(scratch, MEM_OP(reg_x(base.AsXRegister()), offs.Int32Value()));
633   ___ Br(scratch);
634 }
635 
Call(ManagedRegister m_base,Offset offs)636 void Arm64JNIMacroAssembler::Call(ManagedRegister m_base, Offset offs) {
637   Arm64ManagedRegister base = m_base.AsArm64();
638   CHECK(base.IsXRegister()) << base;
639   ___ Ldr(lr, MEM_OP(reg_x(base.AsXRegister()), offs.Int32Value()));
640   ___ Blr(lr);
641 }
642 
CallFromThread(ThreadOffset64 offset)643 void Arm64JNIMacroAssembler::CallFromThread(ThreadOffset64 offset) {
644   // Call *(TR + offset)
645   ___ Ldr(lr, MEM_OP(reg_x(TR), offset.Int32Value()));
646   ___ Blr(lr);
647 }
648 
CreateJObject(ManagedRegister m_out_reg,FrameOffset spilled_reference_offset,ManagedRegister m_in_reg,bool null_allowed)649 void Arm64JNIMacroAssembler::CreateJObject(ManagedRegister m_out_reg,
650                                            FrameOffset spilled_reference_offset,
651                                            ManagedRegister m_in_reg,
652                                            bool null_allowed) {
653   Arm64ManagedRegister out_reg = m_out_reg.AsArm64();
654   Arm64ManagedRegister in_reg = m_in_reg.AsArm64();
655   CHECK(in_reg.IsNoRegister() || in_reg.IsWRegister()) << in_reg;
656   CHECK(out_reg.IsXRegister()) << out_reg;
657   if (null_allowed) {
658     UseScratchRegisterScope temps(asm_.GetVIXLAssembler());
659     Register scratch = temps.AcquireX();
660 
661     // Null values get a jobject value null. Otherwise, the jobject is
662     // the address of the spilled reference.
663     // e.g. out_reg = (in == 0) ? 0 : (SP+spilled_reference_offset)
664     if (in_reg.IsNoRegister()) {
665       in_reg = Arm64ManagedRegister::FromWRegister(out_reg.AsOverlappingWRegister());
666       LoadWFromOffset(kLoadWord, in_reg.AsWRegister(), SP, spilled_reference_offset.Int32Value());
667     }
668     ___ Add(scratch, reg_x(SP), spilled_reference_offset.Int32Value());
669     ___ Cmp(reg_w(in_reg.AsWRegister()), 0);
670     ___ Csel(reg_x(out_reg.AsXRegister()), scratch, xzr, ne);
671   } else {
672     AddConstant(out_reg.AsXRegister(), SP, spilled_reference_offset.Int32Value(), al);
673   }
674 }
675 
CreateJObject(FrameOffset out_off,FrameOffset spilled_reference_offset,bool null_allowed)676 void Arm64JNIMacroAssembler::CreateJObject(FrameOffset out_off,
677                                            FrameOffset spilled_reference_offset,
678                                            bool null_allowed) {
679   UseScratchRegisterScope temps(asm_.GetVIXLAssembler());
680   Register scratch = temps.AcquireX();
681   if (null_allowed) {
682     Register scratch2 = temps.AcquireW();
683     ___ Ldr(scratch2, MEM_OP(reg_x(SP), spilled_reference_offset.Int32Value()));
684     ___ Add(scratch, reg_x(SP), spilled_reference_offset.Int32Value());
685     // Null values get a jobject value null. Otherwise, the jobject is
686     // the address of the spilled reference.
687     // e.g. scratch = (scratch == 0) ? 0 : (SP+spilled_reference_offset)
688     ___ Cmp(scratch2, 0);
689     ___ Csel(scratch, scratch, xzr, ne);
690   } else {
691     ___ Add(scratch, reg_x(SP), spilled_reference_offset.Int32Value());
692   }
693   ___ Str(scratch, MEM_OP(reg_x(SP), out_off.Int32Value()));
694 }
695 
DecodeJNITransitionOrLocalJObject(ManagedRegister m_reg,JNIMacroLabel * slow_path,JNIMacroLabel * resume)696 void Arm64JNIMacroAssembler::DecodeJNITransitionOrLocalJObject(ManagedRegister m_reg,
697                                                                JNIMacroLabel* slow_path,
698                                                                JNIMacroLabel* resume) {
699   constexpr uint64_t kGlobalOrWeakGlobalMask = IndirectReferenceTable::GetGlobalOrWeakGlobalMask();
700   constexpr uint64_t kIndirectRefKindMask = IndirectReferenceTable::GetIndirectRefKindMask();
701   constexpr size_t kGlobalOrWeakGlobalBit = WhichPowerOf2(kGlobalOrWeakGlobalMask);
702   Register reg = reg_w(m_reg.AsArm64().AsWRegister());
703   ___ Tbnz(reg.X(), kGlobalOrWeakGlobalBit, Arm64JNIMacroLabel::Cast(slow_path)->AsArm64());
704   ___ And(reg.X(), reg.X(), ~kIndirectRefKindMask);
705   ___ Cbz(reg.X(), Arm64JNIMacroLabel::Cast(resume)->AsArm64());  // Skip load for null.
706   ___ Ldr(reg, MEM_OP(reg.X()));
707 }
708 
TryToTransitionFromRunnableToNative(JNIMacroLabel * label,ArrayRef<const ManagedRegister> scratch_regs)709 void Arm64JNIMacroAssembler::TryToTransitionFromRunnableToNative(
710     JNIMacroLabel* label, [[maybe_unused]] ArrayRef<const ManagedRegister> scratch_regs) {
711   constexpr uint32_t kNativeStateValue = Thread::StoredThreadStateValue(ThreadState::kNative);
712   constexpr uint32_t kRunnableStateValue = Thread::StoredThreadStateValue(ThreadState::kRunnable);
713   constexpr ThreadOffset64 thread_flags_offset = Thread::ThreadFlagsOffset<kArm64PointerSize>();
714   constexpr ThreadOffset64 thread_held_mutex_mutator_lock_offset =
715       Thread::HeldMutexOffset<kArm64PointerSize>(kMutatorLock);
716 
717   UseScratchRegisterScope temps(asm_.GetVIXLAssembler());
718   Register scratch = temps.AcquireW();
719   Register scratch2 = temps.AcquireW();
720 
721   // CAS release, old_value = kRunnableStateValue, new_value = kNativeStateValue, no flags.
722   vixl::aarch64::Label retry;
723   ___ Bind(&retry);
724   static_assert(thread_flags_offset.Int32Value() == 0);  // LDXR/STLXR require exact address.
725   ___ Ldxr(scratch, MEM_OP(reg_x(TR)));
726   ___ Mov(scratch2, kNativeStateValue);
727   // If any flags are set, go to the slow path.
728   static_assert(kRunnableStateValue == 0u);
729   ___ Cbnz(scratch, Arm64JNIMacroLabel::Cast(label)->AsArm64());
730   ___ Stlxr(scratch, scratch2, MEM_OP(reg_x(TR)));
731   ___ Cbnz(scratch, &retry);
732 
733   // Clear `self->tlsPtr_.held_mutexes[kMutatorLock]`.
734   ___ Str(xzr, MEM_OP(reg_x(TR), thread_held_mutex_mutator_lock_offset.Int32Value()));
735 }
736 
TryToTransitionFromNativeToRunnable(JNIMacroLabel * label,ArrayRef<const ManagedRegister> scratch_regs,ManagedRegister return_reg)737 void Arm64JNIMacroAssembler::TryToTransitionFromNativeToRunnable(
738     JNIMacroLabel* label,
739     [[maybe_unused]] ArrayRef<const ManagedRegister> scratch_regs,
740     [[maybe_unused]] ManagedRegister return_reg) {
741   constexpr uint32_t kNativeStateValue = Thread::StoredThreadStateValue(ThreadState::kNative);
742   constexpr uint32_t kRunnableStateValue = Thread::StoredThreadStateValue(ThreadState::kRunnable);
743   constexpr ThreadOffset64 thread_flags_offset = Thread::ThreadFlagsOffset<kArm64PointerSize>();
744   constexpr ThreadOffset64 thread_held_mutex_mutator_lock_offset =
745       Thread::HeldMutexOffset<kArm64PointerSize>(kMutatorLock);
746   constexpr ThreadOffset64 thread_mutator_lock_offset =
747       Thread::MutatorLockOffset<kArm64PointerSize>();
748 
749   UseScratchRegisterScope temps(asm_.GetVIXLAssembler());
750   Register scratch = temps.AcquireW();
751   Register scratch2 = temps.AcquireW();
752 
753   // CAS acquire, old_value = kNativeStateValue, new_value = kRunnableStateValue, no flags.
754   vixl::aarch64::Label retry;
755   ___ Bind(&retry);
756   static_assert(thread_flags_offset.Int32Value() == 0);  // LDAXR/STXR require exact address.
757   ___ Ldaxr(scratch, MEM_OP(reg_x(TR)));
758   ___ Mov(scratch2, kNativeStateValue);
759   // If any flags are set, or the state is not Native, go to the slow path.
760   // (While the thread can theoretically transition between different Suspended states,
761   // it would be very unexpected to see a state other than Native at this point.)
762   ___ Cmp(scratch, scratch2);
763   ___ B(ne, Arm64JNIMacroLabel::Cast(label)->AsArm64());
764   static_assert(kRunnableStateValue == 0u);
765   ___ Stxr(scratch, wzr, MEM_OP(reg_x(TR)));
766   ___ Cbnz(scratch, &retry);
767 
768   // Set `self->tlsPtr_.held_mutexes[kMutatorLock]` to the mutator lock.
769   ___ Ldr(scratch.X(), MEM_OP(reg_x(TR), thread_mutator_lock_offset.Int32Value()));
770   ___ Str(scratch.X(), MEM_OP(reg_x(TR), thread_held_mutex_mutator_lock_offset.Int32Value()));
771 }
772 
SuspendCheck(JNIMacroLabel * label)773 void Arm64JNIMacroAssembler::SuspendCheck(JNIMacroLabel* label) {
774   UseScratchRegisterScope temps(asm_.GetVIXLAssembler());
775   Register scratch = temps.AcquireW();
776   ___ Ldr(scratch, MEM_OP(reg_x(TR), Thread::ThreadFlagsOffset<kArm64PointerSize>().Int32Value()));
777   ___ Tst(scratch, Thread::SuspendOrCheckpointRequestFlags());
778   ___ B(ne, Arm64JNIMacroLabel::Cast(label)->AsArm64());
779 }
780 
ExceptionPoll(JNIMacroLabel * label)781 void Arm64JNIMacroAssembler::ExceptionPoll(JNIMacroLabel* label) {
782   UseScratchRegisterScope temps(asm_.GetVIXLAssembler());
783   Register scratch = temps.AcquireX();
784   ___ Ldr(scratch, MEM_OP(reg_x(TR), Thread::ExceptionOffset<kArm64PointerSize>().Int32Value()));
785   ___ Cbnz(scratch, Arm64JNIMacroLabel::Cast(label)->AsArm64());
786 }
787 
DeliverPendingException()788 void Arm64JNIMacroAssembler::DeliverPendingException() {
789   // Pass exception object as argument.
790   // Don't care about preserving X0 as this won't return.
791   // Note: The scratch register from `ExceptionPoll()` may have been clobbered.
792   ___ Ldr(reg_x(X0), MEM_OP(reg_x(TR), Thread::ExceptionOffset<kArm64PointerSize>().Int32Value()));
793   ___ Ldr(lr,
794           MEM_OP(reg_x(TR),
795                  QUICK_ENTRYPOINT_OFFSET(kArm64PointerSize, pDeliverException).Int32Value()));
796   ___ Blr(lr);
797   // Call should never return.
798   ___ Brk();
799 }
800 
CreateLabel()801 std::unique_ptr<JNIMacroLabel> Arm64JNIMacroAssembler::CreateLabel() {
802   return std::unique_ptr<JNIMacroLabel>(new (asm_.GetAllocator()) Arm64JNIMacroLabel());
803 }
804 
Jump(JNIMacroLabel * label)805 void Arm64JNIMacroAssembler::Jump(JNIMacroLabel* label) {
806   CHECK(label != nullptr);
807   ___ B(Arm64JNIMacroLabel::Cast(label)->AsArm64());
808 }
809 
TestGcMarking(JNIMacroLabel * label,JNIMacroUnaryCondition cond)810 void Arm64JNIMacroAssembler::TestGcMarking(JNIMacroLabel* label, JNIMacroUnaryCondition cond) {
811   CHECK(label != nullptr);
812 
813   UseScratchRegisterScope temps(asm_.GetVIXLAssembler());
814   Register test_reg;
815   DCHECK_EQ(Thread::IsGcMarkingSize(), 4u);
816   if (kUseBakerReadBarrier) {
817     // TestGcMarking() is used in the JNI stub entry when the marking register is up to date.
818     if (kIsDebugBuild && emit_run_time_checks_in_debug_mode_) {
819       Register temp = temps.AcquireW();
820       asm_.GenerateMarkingRegisterCheck(temp);
821     }
822     test_reg = reg_w(MR);
823   } else {
824     test_reg = temps.AcquireW();
825     int32_t is_gc_marking_offset = Thread::IsGcMarkingOffset<kArm64PointerSize>().Int32Value();
826     ___ Ldr(test_reg, MEM_OP(reg_x(TR), is_gc_marking_offset));
827   }
828   switch (cond) {
829     case JNIMacroUnaryCondition::kZero:
830       ___ Cbz(test_reg, Arm64JNIMacroLabel::Cast(label)->AsArm64());
831       break;
832     case JNIMacroUnaryCondition::kNotZero:
833       ___ Cbnz(test_reg, Arm64JNIMacroLabel::Cast(label)->AsArm64());
834       break;
835   }
836 }
837 
TestMarkBit(ManagedRegister m_ref,JNIMacroLabel * label,JNIMacroUnaryCondition cond)838 void Arm64JNIMacroAssembler::TestMarkBit(ManagedRegister m_ref,
839                                          JNIMacroLabel* label,
840                                          JNIMacroUnaryCondition cond) {
841   DCHECK(kUseBakerReadBarrier);
842   Register ref = reg_x(m_ref.AsArm64().AsOverlappingXRegister());
843   UseScratchRegisterScope temps(asm_.GetVIXLAssembler());
844   Register scratch = temps.AcquireW();
845   ___ Ldr(scratch, MEM_OP(ref, mirror::Object::MonitorOffset().SizeValue()));
846   static_assert(LockWord::kMarkBitStateSize == 1u);
847   switch (cond) {
848     case JNIMacroUnaryCondition::kZero:
849       ___ Tbz(scratch, LockWord::kMarkBitStateShift, Arm64JNIMacroLabel::Cast(label)->AsArm64());
850       break;
851     case JNIMacroUnaryCondition::kNotZero:
852       ___ Tbnz(scratch, LockWord::kMarkBitStateShift, Arm64JNIMacroLabel::Cast(label)->AsArm64());
853       break;
854   }
855 }
856 
TestByteAndJumpIfNotZero(uintptr_t address,JNIMacroLabel * label)857 void Arm64JNIMacroAssembler::TestByteAndJumpIfNotZero(uintptr_t address, JNIMacroLabel* label) {
858   UseScratchRegisterScope temps(asm_.GetVIXLAssembler());
859   Register scratch = temps.AcquireX();
860   ___ Mov(scratch, address);
861   ___ Ldrb(scratch.W(), MEM_OP(scratch, 0));
862   ___ Cbnz(scratch.W(), Arm64JNIMacroLabel::Cast(label)->AsArm64());
863 }
864 
Bind(JNIMacroLabel * label)865 void Arm64JNIMacroAssembler::Bind(JNIMacroLabel* label) {
866   CHECK(label != nullptr);
867   ___ Bind(Arm64JNIMacroLabel::Cast(label)->AsArm64());
868 }
869 
BuildFrame(size_t frame_size,ManagedRegister method_reg,ArrayRef<const ManagedRegister> callee_save_regs)870 void Arm64JNIMacroAssembler::BuildFrame(size_t frame_size,
871                                         ManagedRegister method_reg,
872                                         ArrayRef<const ManagedRegister> callee_save_regs) {
873   // Setup VIXL CPURegList for callee-saves.
874   CPURegList core_reg_list(CPURegister::kRegister, kXRegSize, 0);
875   CPURegList fp_reg_list(CPURegister::kVRegister, kDRegSize, 0);
876   for (auto r : callee_save_regs) {
877     Arm64ManagedRegister reg = r.AsArm64();
878     if (reg.IsXRegister()) {
879       core_reg_list.Combine(reg_x(reg.AsXRegister()).GetCode());
880     } else {
881       DCHECK(reg.IsDRegister());
882       fp_reg_list.Combine(reg_d(reg.AsDRegister()).GetCode());
883     }
884   }
885   size_t core_reg_size = core_reg_list.GetTotalSizeInBytes();
886   size_t fp_reg_size = fp_reg_list.GetTotalSizeInBytes();
887 
888   // Increase frame to required size.
889   DCHECK_ALIGNED(frame_size, kStackAlignment);
890   // Must at least have space for Method* if we're going to spill it.
891   DCHECK_GE(frame_size,
892             core_reg_size + fp_reg_size + (method_reg.IsRegister() ? kXRegSizeInBytes : 0u));
893   IncreaseFrameSize(frame_size);
894 
895   // Save callee-saves.
896   asm_.SpillRegisters(core_reg_list, frame_size - core_reg_size);
897   asm_.SpillRegisters(fp_reg_list, frame_size - core_reg_size - fp_reg_size);
898 
899   if (method_reg.IsRegister()) {
900     // Write ArtMethod*
901     DCHECK(X0 == method_reg.AsArm64().AsXRegister());
902     StoreToOffset(X0, SP, 0);
903   }
904 }
905 
RemoveFrame(size_t frame_size,ArrayRef<const ManagedRegister> callee_save_regs,bool may_suspend)906 void Arm64JNIMacroAssembler::RemoveFrame(size_t frame_size,
907                                          ArrayRef<const ManagedRegister> callee_save_regs,
908                                          bool may_suspend) {
909   // Setup VIXL CPURegList for callee-saves.
910   CPURegList core_reg_list(CPURegister::kRegister, kXRegSize, 0);
911   CPURegList fp_reg_list(CPURegister::kVRegister, kDRegSize, 0);
912   for (auto r : callee_save_regs) {
913     Arm64ManagedRegister reg = r.AsArm64();
914     if (reg.IsXRegister()) {
915       core_reg_list.Combine(reg_x(reg.AsXRegister()).GetCode());
916     } else {
917       DCHECK(reg.IsDRegister());
918       fp_reg_list.Combine(reg_d(reg.AsDRegister()).GetCode());
919     }
920   }
921   size_t core_reg_size = core_reg_list.GetTotalSizeInBytes();
922   size_t fp_reg_size = fp_reg_list.GetTotalSizeInBytes();
923 
924   // For now we only check that the size of the frame is large enough to hold spills and method
925   // reference.
926   DCHECK_GE(frame_size, core_reg_size + fp_reg_size);
927   DCHECK_ALIGNED(frame_size, kAapcs64StackAlignment);
928 
929   cfi().RememberState();
930 
931   // Restore callee-saves.
932   asm_.UnspillRegisters(core_reg_list, frame_size - core_reg_size);
933   asm_.UnspillRegisters(fp_reg_list, frame_size - core_reg_size - fp_reg_size);
934 
935   // Emit marking register refresh even with all GCs as we are still using the
936   // register due to nterp's dependency.
937   if (kReserveMarkingRegister) {
938     vixl::aarch64::Register mr = reg_x(MR);  // Marking Register.
939     vixl::aarch64::Register tr = reg_x(TR);  // Thread Register.
940 
941     if (may_suspend) {
942       // The method may be suspended; refresh the Marking Register.
943       ___ Ldr(mr.W(), MemOperand(tr, Thread::IsGcMarkingOffset<kArm64PointerSize>().Int32Value()));
944     } else {
945       // The method shall not be suspended; no need to refresh the Marking Register.
946 
947       // The Marking Register is a callee-save register and thus has been
948       // preserved by native code following the AAPCS64 calling convention.
949 
950       // The following condition is a compile-time one, so it does not have a run-time cost.
951       if (kIsDebugBuild) {
952         // The following condition is a run-time one; it is executed after the
953         // previous compile-time test, to avoid penalizing non-debug builds.
954         if (emit_run_time_checks_in_debug_mode_) {
955           // Emit a run-time check verifying that the Marking Register is up-to-date.
956           UseScratchRegisterScope temps(asm_.GetVIXLAssembler());
957           Register temp = temps.AcquireW();
958           // Ensure we are not clobbering a callee-save register that was restored before.
959           DCHECK(!core_reg_list.IncludesAliasOf(temp.X()))
960               << "core_reg_list should not contain scratch register X" << temp.GetCode();
961           asm_.GenerateMarkingRegisterCheck(temp);
962         }
963       }
964     }
965   }
966 
967   // Decrease frame size to start of callee saved regs.
968   DecreaseFrameSize(frame_size);
969 
970   // Return to LR.
971   ___ Ret();
972 
973   // The CFI should be restored for any code that follows the exit block.
974   cfi().RestoreState();
975   cfi().DefCFAOffset(frame_size);
976 }
977 
LoadLocalReferenceTableStates(ManagedRegister jni_env_reg,ManagedRegister previous_state_reg,ManagedRegister current_state_reg)978 void Arm64JNIMacroAssembler::LoadLocalReferenceTableStates(ManagedRegister jni_env_reg,
979                                                            ManagedRegister previous_state_reg,
980                                                            ManagedRegister current_state_reg) {
981   constexpr size_t kLRTSegmentStateSize = sizeof(jni::LRTSegmentState);
982   DCHECK_EQ(kLRTSegmentStateSize, kWRegSizeInBytes);
983   const MemberOffset previous_state_offset = JNIEnvExt::LrtPreviousStateOffset(kArm64PointerSize);
984   const MemberOffset current_state_offset = JNIEnvExt::LrtSegmentStateOffset(kArm64PointerSize);
985   DCHECK_EQ(previous_state_offset.SizeValue() + kLRTSegmentStateSize,
986             current_state_offset.SizeValue());
987 
988   ___ Ldp(
989       reg_w(previous_state_reg.AsArm64().AsWRegister()),
990       reg_w(current_state_reg.AsArm64().AsWRegister()),
991       MemOperand(reg_x(jni_env_reg.AsArm64().AsXRegister()), previous_state_offset.Int32Value()));
992 }
993 
StoreLocalReferenceTableStates(ManagedRegister jni_env_reg,ManagedRegister previous_state_reg,ManagedRegister current_state_reg)994 void Arm64JNIMacroAssembler::StoreLocalReferenceTableStates(ManagedRegister jni_env_reg,
995                                                             ManagedRegister previous_state_reg,
996                                                             ManagedRegister current_state_reg) {
997   constexpr size_t kLRTSegmentStateSize = sizeof(jni::LRTSegmentState);
998   DCHECK_EQ(kLRTSegmentStateSize, kWRegSizeInBytes);
999   const MemberOffset previous_state_offset = JNIEnvExt::LrtPreviousStateOffset(kArm64PointerSize);
1000   const MemberOffset current_state_offset = JNIEnvExt::LrtSegmentStateOffset(kArm64PointerSize);
1001   DCHECK_EQ(previous_state_offset.SizeValue() + kLRTSegmentStateSize,
1002             current_state_offset.SizeValue());
1003 
1004   // Set the current segment state together with restoring the cookie.
1005   ___ Stp(
1006       reg_w(previous_state_reg.AsArm64().AsWRegister()),
1007       reg_w(current_state_reg.AsArm64().AsWRegister()),
1008       MemOperand(reg_x(jni_env_reg.AsArm64().AsXRegister()), previous_state_offset.Int32Value()));
1009 }
1010 
1011 #undef ___
1012 
1013 }  // namespace arm64
1014 }  // namespace art
1015