1 /*
2 * Copyright (C) 2015 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "linker/arm/relative_patcher_thumb2.h"
18
19 #include "arch/arm/instruction_set_features_arm.h"
20 #include "base/casts.h"
21 #include "driver/compiler_options.h"
22 #include "linker/relative_patcher_test.h"
23 #include "lock_word.h"
24 #include "mirror/array-inl.h"
25 #include "mirror/object.h"
26 #include "oat/oat_quick_method_header.h"
27 #include "optimizing/code_generator_arm_vixl.h"
28 #include "optimizing/optimizing_unit_test.h"
29
30 namespace art {
31 namespace linker {
32
33 class Thumb2RelativePatcherTest : public RelativePatcherTest {
34 public:
Thumb2RelativePatcherTest()35 Thumb2RelativePatcherTest() : RelativePatcherTest(InstructionSet::kThumb2, "default") { }
36
37 protected:
38 static const uint8_t kCallRawCode[];
39 static const ArrayRef<const uint8_t> kCallCode;
40 static const uint8_t kNopRawCode[];
41 static const ArrayRef<const uint8_t> kNopCode;
42 static const uint8_t kUnpatchedPcRelativeRawCode[];
43 static const ArrayRef<const uint8_t> kUnpatchedPcRelativeCode;
44 static const uint32_t kPcInsnOffset;
45
46 // The PC in Thumb mode is 4 bytes after the instruction location.
47 static constexpr uint32_t kPcAdjustment = 4u;
48
49 // Branches within range [-256, 256) can be created from these by adding the low 8 bits.
50 static constexpr uint32_t kBlPlus0 = 0xf000f800u;
51 static constexpr uint32_t kBlMinus256 = 0xf7ffff00u;
52
53 // Special BL values.
54 static constexpr uint32_t kBlPlusMax = 0xf3ffd7ffu;
55 static constexpr uint32_t kBlMinusMax = 0xf400d000u;
56
57 // BNE +0, 32-bit, encoding T3. Bits 0-10, 11, 13, 16-21, 26 are placeholder for target offset.
58 static constexpr uint32_t kBneWPlus0 = 0xf0408000u;
59
60 // LDR immediate, 16-bit, encoding T1. Bits 6-10 are imm5, 0-2 are Rt, 3-5 are Rn.
61 static constexpr uint32_t kLdrInsn = 0x6800u;
62
63 // LDR immediate, 32-bit, encoding T3. Bits 0-11 are offset, 12-15 are Rt, 16-20 are Rn.
64 static constexpr uint32_t kLdrWInsn = 0xf8d00000u;
65
66 // LDR immediate, negative offset, encoding T4. Bits 0-7 are the offset to subtract.
67 static constexpr uint32_t kLdrNegativeOffset = 0xf8500c00u;
68
69 // LDR register, lsl #2. Bits 4-5 are the imm2, i.e. the lsl shift.
70 static constexpr uint32_t kLdrRegLsl2 = 0xf8500020u;
71
72 // NOP instructions.
73 static constexpr uint32_t kNopInsn = 0xbf00u;
74 static constexpr uint32_t kNopWInsn = 0xf3af8000u;
75
InsertInsn(std::vector<uint8_t> * code,size_t pos,uint32_t insn)76 void InsertInsn(std::vector<uint8_t>* code, size_t pos, uint32_t insn) {
77 CHECK_LE(pos, code->size());
78 if (IsUint<16>(insn)) {
79 const uint8_t insn_code[] = {
80 static_cast<uint8_t>(insn),
81 static_cast<uint8_t>(insn >> 8),
82 };
83 static_assert(sizeof(insn_code) == 2u, "Invalid sizeof(insn_code).");
84 code->insert(code->begin() + pos, insn_code, insn_code + sizeof(insn_code));
85 } else {
86 const uint8_t insn_code[] = {
87 static_cast<uint8_t>(insn >> 16),
88 static_cast<uint8_t>(insn >> 24),
89 static_cast<uint8_t>(insn),
90 static_cast<uint8_t>(insn >> 8),
91 };
92 static_assert(sizeof(insn_code) == 4u, "Invalid sizeof(insn_code).");
93 code->insert(code->begin() + pos, insn_code, insn_code + sizeof(insn_code));
94 }
95 }
96
PushBackInsn(std::vector<uint8_t> * code,uint32_t insn)97 void PushBackInsn(std::vector<uint8_t>* code, uint32_t insn) {
98 InsertInsn(code, code->size(), insn);
99 }
100
GenNops(size_t num_nops)101 std::vector<uint8_t> GenNops(size_t num_nops) {
102 std::vector<uint8_t> result;
103 result.reserve(num_nops * 2u);
104 for (size_t i = 0; i != num_nops; ++i) {
105 PushBackInsn(&result, kNopInsn);
106 }
107 return result;
108 }
109
RawCode(std::initializer_list<uint32_t> insns)110 std::vector<uint8_t> RawCode(std::initializer_list<uint32_t> insns) {
111 std::vector<uint8_t> raw_code;
112 size_t number_of_16_bit_insns =
113 std::count_if(insns.begin(), insns.end(), [](uint32_t x) { return IsUint<16>(x); });
114 raw_code.reserve(insns.size() * 4u - number_of_16_bit_insns * 2u);
115 for (uint32_t insn : insns) {
116 PushBackInsn(&raw_code, insn);
117 }
118 return raw_code;
119 }
120
BneWWithOffset(uint32_t bne_offset,uint32_t target_offset)121 uint32_t BneWWithOffset(uint32_t bne_offset, uint32_t target_offset) {
122 if (!IsAligned<2u>(bne_offset)) {
123 LOG(ERROR) << "Unaligned bne_offset: " << bne_offset;
124 return 0xffffffffu; // Fails code diff later.
125 }
126 if (!IsAligned<2u>(target_offset)) {
127 LOG(ERROR) << "Unaligned target_offset: " << target_offset;
128 return 0xffffffffu; // Fails code diff later.
129 }
130 uint32_t diff = target_offset - bne_offset - kPcAdjustment;
131 DCHECK_ALIGNED(diff, 2u);
132 if ((diff >> 20) != 0 && (diff >> 20) != 0xfffu) {
133 LOG(ERROR) << "Target out of range: " << diff;
134 return 0xffffffffu; // Fails code diff later.
135 }
136 return kBneWPlus0 | ((diff >> 1) & 0x7ffu) // imm11
137 | (((diff >> 12) & 0x3fu) << 16) // imm6
138 | (((diff >> 18) & 1) << 13) // J1
139 | (((diff >> 19) & 1) << 11) // J2
140 | (((diff >> 20) & 1) << 26); // S
141 }
142
Create2MethodsWithGap(const ArrayRef<const uint8_t> & method1_code,const ArrayRef<const LinkerPatch> & method1_patches,const ArrayRef<const uint8_t> & last_method_code,const ArrayRef<const LinkerPatch> & last_method_patches,uint32_t distance_without_thunks)143 uint32_t Create2MethodsWithGap(const ArrayRef<const uint8_t>& method1_code,
144 const ArrayRef<const LinkerPatch>& method1_patches,
145 const ArrayRef<const uint8_t>& last_method_code,
146 const ArrayRef<const LinkerPatch>& last_method_patches,
147 uint32_t distance_without_thunks) {
148 CHECK_EQ(distance_without_thunks % kArmCodeAlignment, 0u);
149 uint32_t method1_offset =
150 kTrampolineSize + CodeAlignmentSize(kTrampolineSize) + sizeof(OatQuickMethodHeader);
151 AddCompiledMethod(MethodRef(1u), method1_code, method1_patches);
152 const uint32_t gap_start = method1_offset + method1_code.size();
153
154 // We want to put the last method at a very precise offset.
155 const uint32_t last_method_offset = method1_offset + distance_without_thunks;
156 CHECK_ALIGNED(last_method_offset, kArmCodeAlignment);
157 const uint32_t gap_end = last_method_offset - sizeof(OatQuickMethodHeader);
158
159 // Fill the gap with intermediate methods in chunks of 2MiB and the first in [2MiB, 4MiB).
160 // (This allows deduplicating the small chunks to avoid using 32MiB of memory for +-16MiB
161 // offsets by this test. Making the first chunk bigger makes it easy to give all intermediate
162 // methods the same alignment of the end, so the thunk insertion adds a predictable size as
163 // long as it's after the first chunk.)
164 uint32_t method_idx = 2u;
165 constexpr uint32_t kSmallChunkSize = 2 * MB;
166 std::vector<uint8_t> gap_code;
167 uint32_t gap_size = gap_end - gap_start;
168 uint32_t num_small_chunks = std::max(gap_size / kSmallChunkSize, 1u) - 1u;
169 uint32_t chunk_start = gap_start;
170 uint32_t chunk_size = gap_size - num_small_chunks * kSmallChunkSize;
171 for (uint32_t i = 0; i <= num_small_chunks; ++i) { // num_small_chunks+1 iterations.
172 uint32_t chunk_code_size =
173 chunk_size - CodeAlignmentSize(chunk_start) - sizeof(OatQuickMethodHeader);
174 gap_code.resize(chunk_code_size, 0u);
175 AddCompiledMethod(MethodRef(method_idx), ArrayRef<const uint8_t>(gap_code));
176 method_idx += 1u;
177 chunk_start += chunk_size;
178 chunk_size = kSmallChunkSize; // For all but the first chunk.
179 DCHECK_EQ(CodeAlignmentSize(gap_end), CodeAlignmentSize(chunk_start));
180 }
181
182 // Add the last method and link
183 AddCompiledMethod(MethodRef(method_idx), last_method_code, last_method_patches);
184 Link();
185
186 // Check assumptions.
187 CHECK_EQ(GetMethodOffset(1), method1_offset);
188 auto last_result = method_offset_map_.FindMethodOffset(MethodRef(method_idx));
189 CHECK(last_result.first);
190 // There may be a thunk before method2.
191 if (last_result.second != last_method_offset + 1 /* thumb mode */) {
192 // Thunk present. Check that there's only one.
193 uint32_t thunk_end =
194 CompiledCode::AlignCode(gap_end, InstructionSet::kThumb2) + MethodCallThunkSize();
195 uint32_t header_offset = thunk_end + CodeAlignmentSize(thunk_end);
196 CHECK_EQ(last_result.second,
197 header_offset + sizeof(OatQuickMethodHeader) + 1 /* thumb mode */);
198 }
199 return method_idx;
200 }
201
GetMethodOffset(uint32_t method_idx)202 uint32_t GetMethodOffset(uint32_t method_idx) {
203 auto result = method_offset_map_.FindMethodOffset(MethodRef(method_idx));
204 CHECK(result.first);
205 CHECK_NE(result.second & 1u, 0u);
206 return result.second - 1 /* thumb mode */;
207 }
208
CompileThunk(const LinkerPatch & patch,std::string * debug_name=nullptr)209 std::vector<uint8_t> CompileThunk(const LinkerPatch& patch,
210 /*out*/ std::string* debug_name = nullptr) {
211 OptimizingUnitTestHelper helper;
212 HGraph* graph = helper.CreateGraph();
213 CompilerOptions compiler_options;
214
215 // Set isa to Thumb2.
216 compiler_options.instruction_set_ = instruction_set_;
217 compiler_options.instruction_set_features_ =
218 InstructionSetFeatures::FromBitmap(instruction_set_, instruction_set_features_->AsBitmap());
219 CHECK(compiler_options.instruction_set_features_->Equals(instruction_set_features_.get()));
220
221 // If a test requests that implicit null checks are enabled or disabled,
222 // apply that option, otherwise use the default from `CompilerOptions`.
223 if (implicit_null_checks_.has_value()) {
224 compiler_options.implicit_null_checks_ = implicit_null_checks_.value();
225 }
226
227 arm::CodeGeneratorARMVIXL codegen(graph, compiler_options);
228 ArenaVector<uint8_t> code(helper.GetAllocator()->Adapter());
229 codegen.EmitThunkCode(patch, &code, debug_name);
230 return std::vector<uint8_t>(code.begin(), code.end());
231 }
232
AddCompiledMethod(MethodReference method_ref,const ArrayRef<const uint8_t> & code,const ArrayRef<const LinkerPatch> & patches=ArrayRef<const LinkerPatch> ())233 void AddCompiledMethod(
234 MethodReference method_ref,
235 const ArrayRef<const uint8_t>& code,
236 const ArrayRef<const LinkerPatch>& patches = ArrayRef<const LinkerPatch>()) {
237 RelativePatcherTest::AddCompiledMethod(method_ref, code, patches);
238
239 // Make sure the ThunkProvider has all the necessary thunks.
240 for (const LinkerPatch& patch : patches) {
241 if (patch.GetType() == LinkerPatch::Type::kCallEntrypoint ||
242 patch.GetType() == LinkerPatch::Type::kBakerReadBarrierBranch ||
243 patch.GetType() == LinkerPatch::Type::kCallRelative) {
244 std::string debug_name;
245 std::vector<uint8_t> thunk_code = CompileThunk(patch, &debug_name);
246 thunk_provider_.SetThunkCode(patch, ArrayRef<const uint8_t>(thunk_code), debug_name);
247 }
248 }
249 }
250
CompileMethodCallThunk()251 std::vector<uint8_t> CompileMethodCallThunk() {
252 LinkerPatch patch = LinkerPatch::RelativeCodePatch(/* literal_offset */ 0u,
253 /* target_dex_file*/ nullptr,
254 /* target_method_idx */ 0u);
255 return CompileThunk(patch);
256 }
257
MethodCallThunkSize()258 uint32_t MethodCallThunkSize() {
259 return CompileMethodCallThunk().size();
260 }
261
CheckThunk(uint32_t thunk_offset)262 bool CheckThunk(uint32_t thunk_offset) {
263 const std::vector<uint8_t> expected_code = CompileMethodCallThunk();
264 if (output_.size() < thunk_offset + expected_code.size()) {
265 LOG(ERROR) << "output_.size() == " << output_.size() << " < "
266 << "thunk_offset + expected_code.size() == " << (thunk_offset + expected_code.size());
267 return false;
268 }
269 ArrayRef<const uint8_t> linked_code(&output_[thunk_offset], expected_code.size());
270 if (linked_code == ArrayRef<const uint8_t>(expected_code)) {
271 return true;
272 }
273 // Log failure info.
274 DumpDiff(ArrayRef<const uint8_t>(expected_code), linked_code);
275 return false;
276 }
277
GenNopsAndBl(size_t num_nops,uint32_t bl)278 std::vector<uint8_t> GenNopsAndBl(size_t num_nops, uint32_t bl) {
279 std::vector<uint8_t> result;
280 result.reserve(num_nops * 2u + 4u);
281 for (size_t i = 0; i != num_nops; ++i) {
282 PushBackInsn(&result, kNopInsn);
283 }
284 PushBackInsn(&result, bl);
285 return result;
286 }
287
288 void TestStringBssEntry(uint32_t bss_begin, uint32_t string_entry_offset);
289 void TestStringReference(uint32_t string_offset);
290 void CheckPcRelativePatch(const ArrayRef<const LinkerPatch>& patches, uint32_t target_offset);
291
EncodeBakerReadBarrierFieldData(uint32_t base_reg,uint32_t holder_reg,bool narrow)292 static uint32_t EncodeBakerReadBarrierFieldData(uint32_t base_reg,
293 uint32_t holder_reg,
294 bool narrow) {
295 return arm::CodeGeneratorARMVIXL::EncodeBakerReadBarrierFieldData(base_reg, holder_reg, narrow);
296 }
297
EncodeBakerReadBarrierArrayData(uint32_t base_reg)298 static uint32_t EncodeBakerReadBarrierArrayData(uint32_t base_reg) {
299 return arm::CodeGeneratorARMVIXL::EncodeBakerReadBarrierArrayData(base_reg);
300 }
301
EncodeBakerReadBarrierGcRootData(uint32_t root_reg,bool narrow)302 static uint32_t EncodeBakerReadBarrierGcRootData(uint32_t root_reg, bool narrow) {
303 return arm::CodeGeneratorARMVIXL::EncodeBakerReadBarrierGcRootData(root_reg, narrow);
304 }
305
CompileBakerOffsetThunk(uint32_t base_reg,uint32_t holder_reg,bool narrow)306 std::vector<uint8_t> CompileBakerOffsetThunk(uint32_t base_reg,
307 uint32_t holder_reg,
308 bool narrow) {
309 const LinkerPatch patch = LinkerPatch::BakerReadBarrierBranchPatch(
310 /* literal_offset */ 0u, EncodeBakerReadBarrierFieldData(base_reg, holder_reg, narrow));
311 return CompileThunk(patch);
312 }
313
CompileBakerArrayThunk(uint32_t base_reg)314 std::vector<uint8_t> CompileBakerArrayThunk(uint32_t base_reg) {
315 LinkerPatch patch = LinkerPatch::BakerReadBarrierBranchPatch(
316 /* literal_offset */ 0u, EncodeBakerReadBarrierArrayData(base_reg));
317 return CompileThunk(patch);
318 }
319
CompileBakerGcRootThunk(uint32_t root_reg,bool narrow)320 std::vector<uint8_t> CompileBakerGcRootThunk(uint32_t root_reg, bool narrow) {
321 LinkerPatch patch = LinkerPatch::BakerReadBarrierBranchPatch(
322 /* literal_offset */ 0u, EncodeBakerReadBarrierGcRootData(root_reg, narrow));
323 return CompileThunk(patch);
324 }
325
GetOutputInsn32(uint32_t offset)326 uint32_t GetOutputInsn32(uint32_t offset) {
327 CHECK_LE(offset, output_.size());
328 CHECK_GE(output_.size() - offset, 4u);
329 return (static_cast<uint32_t>(output_[offset]) << 16) |
330 (static_cast<uint32_t>(output_[offset + 1]) << 24) |
331 (static_cast<uint32_t>(output_[offset + 2]) << 0) |
332 (static_cast<uint32_t>(output_[offset + 3]) << 8);
333 }
334
GetOutputInsn16(uint32_t offset)335 uint16_t GetOutputInsn16(uint32_t offset) {
336 CHECK_LE(offset, output_.size());
337 CHECK_GE(output_.size() - offset, 2u);
338 return (static_cast<uint32_t>(output_[offset]) << 0) |
339 (static_cast<uint32_t>(output_[offset + 1]) << 8);
340 }
341
342 void TestBakerFieldWide(uint32_t offset, uint32_t ref_reg, bool implicit_null_checks);
343 void TestBakerFieldNarrow(uint32_t offset, uint32_t ref_reg, bool implicit_null_checks);
344
Reset()345 void Reset() final {
346 RelativePatcherTest::Reset();
347 implicit_null_checks_ = std::nullopt;
348 }
349
350 private:
351 std::optional<bool> implicit_null_checks_ = std::nullopt;
352 };
353
354 const uint8_t Thumb2RelativePatcherTest::kCallRawCode[] = {
355 0x00, 0xf0, 0x00, 0xf8
356 };
357
358 const ArrayRef<const uint8_t> Thumb2RelativePatcherTest::kCallCode(kCallRawCode);
359
360 const uint8_t Thumb2RelativePatcherTest::kNopRawCode[] = {
361 0x00, 0xbf
362 };
363
364 const ArrayRef<const uint8_t> Thumb2RelativePatcherTest::kNopCode(kNopRawCode);
365
366 const uint8_t Thumb2RelativePatcherTest::kUnpatchedPcRelativeRawCode[] = {
367 0x40, 0xf2, 0x00, 0x00, // MOVW r0, #0 (placeholder)
368 0xc0, 0xf2, 0x00, 0x00, // MOVT r0, #0 (placeholder)
369 0x78, 0x44, // ADD r0, pc
370 };
371 const ArrayRef<const uint8_t> Thumb2RelativePatcherTest::kUnpatchedPcRelativeCode(
372 kUnpatchedPcRelativeRawCode);
373 const uint32_t Thumb2RelativePatcherTest::kPcInsnOffset = 8u;
374
TestStringBssEntry(uint32_t bss_begin,uint32_t string_entry_offset)375 void Thumb2RelativePatcherTest::TestStringBssEntry(uint32_t bss_begin,
376 uint32_t string_entry_offset) {
377 constexpr uint32_t kStringIndex = 1u;
378 string_index_to_offset_map_.Put(kStringIndex, string_entry_offset);
379 bss_begin_ = bss_begin;
380 const LinkerPatch patches[] = {
381 LinkerPatch::StringBssEntryPatch(0u, nullptr, kPcInsnOffset, kStringIndex),
382 LinkerPatch::StringBssEntryPatch(4u, nullptr, kPcInsnOffset, kStringIndex),
383 };
384 CheckPcRelativePatch(ArrayRef<const LinkerPatch>(patches), bss_begin_ + string_entry_offset);
385 }
386
TestStringReference(uint32_t string_offset)387 void Thumb2RelativePatcherTest::TestStringReference(uint32_t string_offset) {
388 constexpr uint32_t kStringIndex = 1u;
389 string_index_to_offset_map_.Put(kStringIndex, string_offset);
390 const LinkerPatch patches[] = {
391 LinkerPatch::RelativeStringPatch(0u, nullptr, kPcInsnOffset, kStringIndex),
392 LinkerPatch::RelativeStringPatch(4u, nullptr, kPcInsnOffset, kStringIndex),
393 };
394 CheckPcRelativePatch(ArrayRef<const LinkerPatch>(patches), string_offset);
395 }
396
CheckPcRelativePatch(const ArrayRef<const LinkerPatch> & patches,uint32_t target_offset)397 void Thumb2RelativePatcherTest::CheckPcRelativePatch(const ArrayRef<const LinkerPatch>& patches,
398 uint32_t target_offset) {
399 AddCompiledMethod(MethodRef(1u), kUnpatchedPcRelativeCode, ArrayRef<const LinkerPatch>(patches));
400 Link();
401
402 uint32_t method1_offset = GetMethodOffset(1u);
403 uint32_t pc_base_offset = method1_offset + kPcInsnOffset + 4u /* PC adjustment */;
404 uint32_t diff = target_offset - pc_base_offset;
405 // Distribute the bits of the diff between the MOVW and MOVT:
406 uint32_t diffw = diff & 0xffffu;
407 uint32_t difft = diff >> 16;
408 uint32_t movw = 0xf2400000u | // MOVW r0, #0 (placeholder),
409 ((diffw & 0xf000u) << (16 - 12)) | // move imm4 from bits 12-15 to bits 16-19,
410 ((diffw & 0x0800u) << (26 - 11)) | // move imm from bit 11 to bit 26,
411 ((diffw & 0x0700u) << (12 - 8)) | // move imm3 from bits 8-10 to bits 12-14,
412 ((diffw & 0x00ffu)); // keep imm8 at bits 0-7.
413 uint32_t movt = 0xf2c00000u | // MOVT r0, #0 (placeholder),
414 ((difft & 0xf000u) << (16 - 12)) | // move imm4 from bits 12-15 to bits 16-19,
415 ((difft & 0x0800u) << (26 - 11)) | // move imm from bit 11 to bit 26,
416 ((difft & 0x0700u) << (12 - 8)) | // move imm3 from bits 8-10 to bits 12-14,
417 ((difft & 0x00ffu)); // keep imm8 at bits 0-7.
418 const uint8_t expected_code[] = {
419 static_cast<uint8_t>(movw >> 16), static_cast<uint8_t>(movw >> 24),
420 static_cast<uint8_t>(movw >> 0), static_cast<uint8_t>(movw >> 8),
421 static_cast<uint8_t>(movt >> 16), static_cast<uint8_t>(movt >> 24),
422 static_cast<uint8_t>(movt >> 0), static_cast<uint8_t>(movt >> 8),
423 0x78, 0x44,
424 };
425 EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(expected_code)));
426 }
427
TEST_F(Thumb2RelativePatcherTest,CallSelf)428 TEST_F(Thumb2RelativePatcherTest, CallSelf) {
429 const LinkerPatch patches[] = {
430 LinkerPatch::RelativeCodePatch(0u, nullptr, 1u),
431 };
432 AddCompiledMethod(MethodRef(1u), kCallCode, ArrayRef<const LinkerPatch>(patches));
433 Link();
434
435 static const uint8_t expected_code[] = {
436 0xff, 0xf7, 0xfe, 0xff
437 };
438 EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(expected_code)));
439 }
440
TEST_F(Thumb2RelativePatcherTest,CallOther)441 TEST_F(Thumb2RelativePatcherTest, CallOther) {
442 const LinkerPatch method1_patches[] = {
443 LinkerPatch::RelativeCodePatch(0u, nullptr, 2u),
444 };
445 AddCompiledMethod(MethodRef(1u), kCallCode, ArrayRef<const LinkerPatch>(method1_patches));
446 const LinkerPatch method2_patches[] = {
447 LinkerPatch::RelativeCodePatch(0u, nullptr, 1u),
448 };
449 AddCompiledMethod(MethodRef(2u), kCallCode, ArrayRef<const LinkerPatch>(method2_patches));
450 Link();
451
452 uint32_t method1_offset = GetMethodOffset(1u);
453 uint32_t method2_offset = GetMethodOffset(2u);
454 uint32_t diff_after = method2_offset - (method1_offset + 4u /* PC adjustment */);
455 ASSERT_EQ(diff_after & 1u, 0u);
456 ASSERT_LT(diff_after >> 1, 1u << 8); // Simple encoding, (diff_after >> 1) fits into 8 bits.
457 static const uint8_t method1_expected_code[] = {
458 0x00, 0xf0, static_cast<uint8_t>(diff_after >> 1), 0xf8
459 };
460 EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(method1_expected_code)));
461 uint32_t diff_before = method1_offset - (method2_offset + 4u /* PC adjustment */);
462 ASSERT_EQ(diff_before & 1u, 0u);
463 ASSERT_GE(diff_before, -1u << 9); // Simple encoding, -256 <= (diff >> 1) < 0.
464 auto method2_expected_code = GenNopsAndBl(0u, kBlMinus256 | ((diff_before >> 1) & 0xffu));
465 EXPECT_TRUE(CheckLinkedMethod(MethodRef(2u), ArrayRef<const uint8_t>(method2_expected_code)));
466 }
467
TEST_F(Thumb2RelativePatcherTest,CallTrampoline)468 TEST_F(Thumb2RelativePatcherTest, CallTrampoline) {
469 const LinkerPatch patches[] = {
470 LinkerPatch::RelativeCodePatch(0u, nullptr, 2u),
471 };
472 AddCompiledMethod(MethodRef(1u), kCallCode, ArrayRef<const LinkerPatch>(patches));
473 Link();
474
475 uint32_t method1_offset = GetMethodOffset(1u);
476 uint32_t diff = kTrampolineOffset - (method1_offset + 4u);
477 ASSERT_EQ(diff & 1u, 0u);
478 ASSERT_GE(diff, -1u << 9); // Simple encoding, -256 <= (diff >> 1) < 0 (checked as unsigned).
479 auto expected_code = GenNopsAndBl(0u, kBlMinus256 | ((diff >> 1) & 0xffu));
480 EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(expected_code)));
481 }
482
TEST_F(Thumb2RelativePatcherTest,CallTrampolineTooFar)483 TEST_F(Thumb2RelativePatcherTest, CallTrampolineTooFar) {
484 constexpr uint32_t missing_method_index = 1024u;
485 auto last_method_raw_code = GenNopsAndBl(3u, kBlPlus0);
486 constexpr uint32_t bl_offset_in_last_method = 3u * 2u; // After NOPs.
487 ArrayRef<const uint8_t> last_method_code(last_method_raw_code);
488 ASSERT_EQ(bl_offset_in_last_method + 4u, last_method_code.size());
489 const LinkerPatch last_method_patches[] = {
490 LinkerPatch::RelativeCodePatch(bl_offset_in_last_method, nullptr, missing_method_index),
491 };
492
493 constexpr uint32_t just_over_max_negative_disp = 16 * MB + 2 - 4u /* PC adjustment */;
494 uint32_t last_method_idx = Create2MethodsWithGap(
495 kNopCode,
496 ArrayRef<const LinkerPatch>(),
497 last_method_code,
498 ArrayRef<const LinkerPatch>(last_method_patches),
499 just_over_max_negative_disp - bl_offset_in_last_method);
500 uint32_t method1_offset = GetMethodOffset(1u);
501 uint32_t last_method_offset = GetMethodOffset(last_method_idx);
502 ASSERT_EQ(method1_offset,
503 last_method_offset + bl_offset_in_last_method - just_over_max_negative_disp);
504 ASSERT_FALSE(method_offset_map_.FindMethodOffset(MethodRef(missing_method_index)).first);
505
506 // Check linked code.
507 uint32_t thunk_offset = CompiledCode::AlignCode(
508 last_method_offset + last_method_code.size(), InstructionSet::kThumb2);
509 uint32_t diff =
510 thunk_offset - (last_method_offset + bl_offset_in_last_method + 4u /* PC adjustment */);
511 ASSERT_TRUE(IsAligned<2u>(diff));
512 ASSERT_LT(diff >> 1, 1u << 8); // Simple encoding, (diff >> 1) fits into 8 bits.
513 auto expected_code = GenNopsAndBl(3u, kBlPlus0 | ((diff >> 1) & 0xffu));
514 EXPECT_TRUE(CheckLinkedMethod(MethodRef(last_method_idx),
515 ArrayRef<const uint8_t>(expected_code)));
516 EXPECT_TRUE(CheckThunk(thunk_offset));
517 }
518
TEST_F(Thumb2RelativePatcherTest,CallOtherAlmostTooFarAfter)519 TEST_F(Thumb2RelativePatcherTest, CallOtherAlmostTooFarAfter) {
520 auto method1_raw_code = GenNopsAndBl(3u, kBlPlus0);
521 constexpr uint32_t bl_offset_in_method1 = 3u * 2u; // After NOPs.
522 ArrayRef<const uint8_t> method1_code(method1_raw_code);
523 ASSERT_EQ(bl_offset_in_method1 + 4u, method1_code.size());
524 const uint32_t kExpectedLastMethodIdx = 9u; // Based on 2MiB chunks in Create2MethodsWithGap().
525 const LinkerPatch method1_patches[] = {
526 LinkerPatch::RelativeCodePatch(bl_offset_in_method1, nullptr, kExpectedLastMethodIdx),
527 };
528
529 constexpr uint32_t max_positive_disp = 16 * MB - 2u + 4u /* PC adjustment */;
530 uint32_t last_method_idx = Create2MethodsWithGap(method1_code,
531 ArrayRef<const LinkerPatch>(method1_patches),
532 kNopCode,
533 ArrayRef<const LinkerPatch>(),
534 bl_offset_in_method1 + max_positive_disp);
535 ASSERT_EQ(kExpectedLastMethodIdx, last_method_idx);
536
537 // Check linked code.
538 auto expected_code = GenNopsAndBl(3u, kBlPlusMax);
539 EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(expected_code)));
540 }
541
TEST_F(Thumb2RelativePatcherTest,CallOtherAlmostTooFarBefore)542 TEST_F(Thumb2RelativePatcherTest, CallOtherAlmostTooFarBefore) {
543 auto last_method_raw_code = GenNopsAndBl(2u, kBlPlus0);
544 constexpr uint32_t bl_offset_in_last_method = 2u * 2u; // After NOPs.
545 ArrayRef<const uint8_t> last_method_code(last_method_raw_code);
546 ASSERT_EQ(bl_offset_in_last_method + 4u, last_method_code.size());
547 const LinkerPatch last_method_patches[] = {
548 LinkerPatch::RelativeCodePatch(bl_offset_in_last_method, nullptr, 1u),
549 };
550
551 constexpr uint32_t max_negative_disp = 16 * MB - 4u /* PC adjustment */;
552 uint32_t last_method_idx = Create2MethodsWithGap(kNopCode,
553 ArrayRef<const LinkerPatch>(),
554 last_method_code,
555 ArrayRef<const LinkerPatch>(last_method_patches),
556 max_negative_disp - bl_offset_in_last_method);
557 uint32_t method1_offset = GetMethodOffset(1u);
558 uint32_t last_method_offset = GetMethodOffset(last_method_idx);
559 ASSERT_EQ(method1_offset, last_method_offset + bl_offset_in_last_method - max_negative_disp);
560
561 // Check linked code.
562 auto expected_code = GenNopsAndBl(2u, kBlMinusMax);
563 EXPECT_TRUE(CheckLinkedMethod(MethodRef(last_method_idx),
564 ArrayRef<const uint8_t>(expected_code)));
565 }
566
TEST_F(Thumb2RelativePatcherTest,CallOtherJustTooFarAfter)567 TEST_F(Thumb2RelativePatcherTest, CallOtherJustTooFarAfter) {
568 auto method1_raw_code = GenNopsAndBl(2u, kBlPlus0);
569 constexpr uint32_t bl_offset_in_method1 = 2u * 2u; // After NOPs.
570 ArrayRef<const uint8_t> method1_code(method1_raw_code);
571 ASSERT_EQ(bl_offset_in_method1 + 4u, method1_code.size());
572 const uint32_t kExpectedLastMethodIdx = 9u; // Based on 2MiB chunks in Create2MethodsWithGap().
573 const LinkerPatch method1_patches[] = {
574 LinkerPatch::RelativeCodePatch(bl_offset_in_method1, nullptr, kExpectedLastMethodIdx),
575 };
576
577 constexpr uint32_t just_over_max_positive_disp = 16 * MB + 4u /* PC adjustment */;
578 uint32_t last_method_idx = Create2MethodsWithGap(
579 method1_code,
580 ArrayRef<const LinkerPatch>(method1_patches),
581 kNopCode,
582 ArrayRef<const LinkerPatch>(),
583 bl_offset_in_method1 + just_over_max_positive_disp);
584 ASSERT_EQ(kExpectedLastMethodIdx, last_method_idx);
585 uint32_t method_after_thunk_idx = last_method_idx;
586 if (sizeof(OatQuickMethodHeader) < kArmCodeAlignment) {
587 // The thunk needs to start on a kArmCodeAlignment-aligned address before the address where the
588 // last method would have been if there was no thunk. If the size of the OatQuickMethodHeader
589 // is at least kArmCodeAlignment, the thunk start shall fit between the previous filler method
590 // and that address. Otherwise, it shall be inserted before that filler method.
591 method_after_thunk_idx -= 1u;
592 }
593
594 uint32_t method1_offset = GetMethodOffset(1u);
595 uint32_t method_after_thunk_offset = GetMethodOffset(method_after_thunk_idx);
596 ASSERT_TRUE(IsAligned<kArmCodeAlignment>(method_after_thunk_offset));
597 uint32_t method_after_thunk_header_offset =
598 method_after_thunk_offset - sizeof(OatQuickMethodHeader);
599 uint32_t thunk_size = MethodCallThunkSize();
600 uint32_t thunk_offset =
601 RoundDown(method_after_thunk_header_offset - thunk_size, kArmCodeAlignment);
602 DCHECK_EQ(thunk_offset + thunk_size + CodeAlignmentSize(thunk_offset + thunk_size),
603 method_after_thunk_header_offset);
604 ASSERT_TRUE(IsAligned<kArmCodeAlignment>(thunk_offset));
605 uint32_t diff = thunk_offset - (method1_offset + bl_offset_in_method1 + 4u /* PC adjustment */);
606 ASSERT_TRUE(IsAligned<2u>(diff));
607 ASSERT_GE(diff, 16 * MB - (1u << 22)); // Simple encoding, unknown bits fit into imm10:imm11:0.
608 auto expected_code =
609 GenNopsAndBl(2u, 0xf000d000 | ((diff >> 1) & 0x7ffu) | (((diff >> 12) & 0x3ffu) << 16));
610 EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(expected_code)));
611 CheckThunk(thunk_offset);
612 }
613
TEST_F(Thumb2RelativePatcherTest,CallOtherJustTooFarBefore)614 TEST_F(Thumb2RelativePatcherTest, CallOtherJustTooFarBefore) {
615 auto last_method_raw_code = GenNopsAndBl(3u, kBlPlus0);
616 constexpr uint32_t bl_offset_in_last_method = 3u * 2u; // After NOPs.
617 ArrayRef<const uint8_t> last_method_code(last_method_raw_code);
618 ASSERT_EQ(bl_offset_in_last_method + 4u, last_method_code.size());
619 const LinkerPatch last_method_patches[] = {
620 LinkerPatch::RelativeCodePatch(bl_offset_in_last_method, nullptr, 1u),
621 };
622
623 constexpr uint32_t just_over_max_negative_disp = 16 * MB + 2 - 4u /* PC adjustment */;
624 uint32_t last_method_idx = Create2MethodsWithGap(
625 kNopCode,
626 ArrayRef<const LinkerPatch>(),
627 last_method_code,
628 ArrayRef<const LinkerPatch>(last_method_patches),
629 just_over_max_negative_disp - bl_offset_in_last_method);
630 uint32_t method1_offset = GetMethodOffset(1u);
631 uint32_t last_method_offset = GetMethodOffset(last_method_idx);
632 ASSERT_EQ(method1_offset,
633 last_method_offset + bl_offset_in_last_method - just_over_max_negative_disp);
634
635 // Check linked code.
636 uint32_t thunk_offset = CompiledCode::AlignCode(
637 last_method_offset + last_method_code.size(), InstructionSet::kThumb2);
638 uint32_t diff =
639 thunk_offset - (last_method_offset + bl_offset_in_last_method + 4u /* PC adjustment */);
640 ASSERT_TRUE(IsAligned<2u>(diff));
641 ASSERT_LT(diff >> 1, 1u << 8); // Simple encoding, (diff >> 1) fits into 8 bits.
642 auto expected_code = GenNopsAndBl(3u, kBlPlus0 | ((diff >> 1) & 0xffu));
643 EXPECT_TRUE(CheckLinkedMethod(MethodRef(last_method_idx),
644 ArrayRef<const uint8_t>(expected_code)));
645 EXPECT_TRUE(CheckThunk(thunk_offset));
646 }
647
TEST_F(Thumb2RelativePatcherTest,StringBssEntry1)648 TEST_F(Thumb2RelativePatcherTest, StringBssEntry1) {
649 TestStringBssEntry(0x00ff0000u, 0x00fcu);
650 ASSERT_LT(GetMethodOffset(1u), 0xfcu);
651 }
652
TEST_F(Thumb2RelativePatcherTest,StringBssEntry2)653 TEST_F(Thumb2RelativePatcherTest, StringBssEntry2) {
654 TestStringBssEntry(0x02ff0000u, 0x05fcu);
655 ASSERT_LT(GetMethodOffset(1u), 0xfcu);
656 }
657
TEST_F(Thumb2RelativePatcherTest,StringBssEntry3)658 TEST_F(Thumb2RelativePatcherTest, StringBssEntry3) {
659 TestStringBssEntry(0x08ff0000u, 0x08fcu);
660 ASSERT_LT(GetMethodOffset(1u), 0xfcu);
661 }
662
TEST_F(Thumb2RelativePatcherTest,StringBssEntry4)663 TEST_F(Thumb2RelativePatcherTest, StringBssEntry4) {
664 TestStringBssEntry(0xd0ff0000u, 0x60fcu);
665 ASSERT_LT(GetMethodOffset(1u), 0xfcu);
666 }
667
TEST_F(Thumb2RelativePatcherTest,StringReference1)668 TEST_F(Thumb2RelativePatcherTest, StringReference1) {
669 TestStringReference(0x00ff00fcu);
670 ASSERT_LT(GetMethodOffset(1u), 0xfcu);
671 }
672
TEST_F(Thumb2RelativePatcherTest,StringReference2)673 TEST_F(Thumb2RelativePatcherTest, StringReference2) {
674 TestStringReference(0x02ff05fcu);
675 ASSERT_LT(GetMethodOffset(1u), 0xfcu);
676 }
677
TEST_F(Thumb2RelativePatcherTest,StringReference3)678 TEST_F(Thumb2RelativePatcherTest, StringReference3) {
679 TestStringReference(0x08ff08fcu);
680 ASSERT_LT(GetMethodOffset(1u), 0xfcu);
681 }
682
TEST_F(Thumb2RelativePatcherTest,StringReference4)683 TEST_F(Thumb2RelativePatcherTest, StringReference4) {
684 TestStringReference(0xd0ff60fcu);
685 ASSERT_LT(GetMethodOffset(1u), 0xfcu);
686 }
687
TEST_F(Thumb2RelativePatcherTest,EntrypointCall)688 TEST_F(Thumb2RelativePatcherTest, EntrypointCall) {
689 constexpr uint32_t kEntrypointOffset = 512;
690 const LinkerPatch patches[] = {
691 LinkerPatch::CallEntrypointPatch(0u, kEntrypointOffset),
692 };
693 AddCompiledMethod(MethodRef(1u), kCallCode, ArrayRef<const LinkerPatch>(patches));
694 Link();
695
696 uint32_t method_offset = GetMethodOffset(1u);
697 uint32_t thunk_offset = CompiledCode::AlignCode(method_offset + kCallCode.size(),
698 InstructionSet::kThumb2);
699 uint32_t diff = thunk_offset - method_offset - kPcAdjustment;
700 ASSERT_TRUE(IsAligned<2u>(diff));
701 ASSERT_LT(diff >> 1, 1u << 8); // Simple encoding, (diff >> 1) fits into 8 bits.
702 auto expected_code = GenNopsAndBl(0u, kBlPlus0 | ((diff >> 1) & 0xffu));
703 EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(expected_code)));
704
705 // Verify the thunk.
706 uint32_t ldr_pc_tr_offset =
707 0xf8d00000 | // LDR Rt, [Rn, #<imm12>]
708 (/* tr */ 9 << 16) | // Rn = TR
709 (/* pc */ 15 << 12) | // Rt = PC
710 kEntrypointOffset; // imm12
711 uint16_t bkpt = 0xbe00;
712 ASSERT_LE(6u, output_.size() - thunk_offset);
713 EXPECT_EQ(ldr_pc_tr_offset, GetOutputInsn32(thunk_offset));
714 EXPECT_EQ(bkpt, GetOutputInsn16(thunk_offset + 4u));
715 }
716
717 const uint32_t kBakerValidRegs[] = {
718 0, 1, 2, 3, 4, 5, 6, 7,
719 9, 10, 11, // r8 (rMR), IP, SP, LR and PC are reserved.
720 };
721
722 const uint32_t kBakerValidRegsNarrow[] = {
723 0, 1, 2, 3, 4, 5, 6, 7,
724 };
725
TestBakerFieldWide(uint32_t offset,uint32_t ref_reg,bool implicit_null_checks)726 void Thumb2RelativePatcherTest::TestBakerFieldWide(uint32_t offset,
727 uint32_t ref_reg,
728 bool implicit_null_checks) {
729 implicit_null_checks_ = implicit_null_checks;
730 DCHECK_ALIGNED(offset, 4u);
731 DCHECK_LT(offset, 4 * KB);
732 constexpr size_t kMethodCodeSize = 8u;
733 constexpr size_t kLiteralOffset = 0u;
734 uint32_t method_idx = 0u;
735 for (uint32_t base_reg : kBakerValidRegs) {
736 for (uint32_t holder_reg : kBakerValidRegs) {
737 uint32_t ldr = kLdrWInsn | offset | (base_reg << 16) | (ref_reg << 12);
738 const std::vector<uint8_t> raw_code = RawCode({kBneWPlus0, ldr});
739 ASSERT_EQ(kMethodCodeSize, raw_code.size());
740 ArrayRef<const uint8_t> code(raw_code);
741 uint32_t encoded_data = EncodeBakerReadBarrierFieldData(
742 base_reg, holder_reg, /* narrow */ false);
743 const LinkerPatch patches[] = {
744 LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset, encoded_data),
745 };
746 ++method_idx;
747 AddCompiledMethod(MethodRef(method_idx), code, ArrayRef<const LinkerPatch>(patches));
748 }
749 }
750 Link();
751
752 // All thunks are at the end.
753 uint32_t thunk_offset = GetMethodOffset(method_idx) + RoundUp(kMethodCodeSize, kArmCodeAlignment);
754 method_idx = 0u;
755 for (uint32_t base_reg : kBakerValidRegs) {
756 for (uint32_t holder_reg : kBakerValidRegs) {
757 ++method_idx;
758 uint32_t bne = BneWWithOffset(GetMethodOffset(method_idx) + kLiteralOffset, thunk_offset);
759 uint32_t ldr = kLdrWInsn | offset | (base_reg << 16) | (ref_reg << 12);
760 const std::vector<uint8_t> expected_code = RawCode({bne, ldr});
761 ASSERT_EQ(kMethodCodeSize, expected_code.size()) << "bne=0x" << std::hex << bne;
762 ASSERT_TRUE(
763 CheckLinkedMethod(MethodRef(method_idx), ArrayRef<const uint8_t>(expected_code)));
764
765 std::vector<uint8_t> expected_thunk =
766 CompileBakerOffsetThunk(base_reg, holder_reg, /* narrow */ false);
767 ASSERT_GT(output_.size(), thunk_offset);
768 ASSERT_GE(output_.size() - thunk_offset, expected_thunk.size());
769 ArrayRef<const uint8_t> compiled_thunk(output_.data() + thunk_offset,
770 expected_thunk.size());
771 if (ArrayRef<const uint8_t>(expected_thunk) != compiled_thunk) {
772 DumpDiff(ArrayRef<const uint8_t>(expected_thunk), compiled_thunk);
773 ASSERT_TRUE(false);
774 }
775
776 size_t gray_check_offset = thunk_offset;
777 if (implicit_null_checks && holder_reg == base_reg) {
778 // Verify that the null-check uses the correct register, i.e. holder_reg.
779 if (holder_reg < 8) {
780 ASSERT_GE(output_.size() - gray_check_offset, 2u);
781 ASSERT_EQ(0xb100 | holder_reg, GetOutputInsn16(thunk_offset) & 0xfd07u);
782 gray_check_offset +=2u;
783 } else {
784 ASSERT_GE(output_.size() - gray_check_offset, 6u);
785 ASSERT_EQ(0xf1b00f00u | (holder_reg << 16), GetOutputInsn32(thunk_offset) & 0xfbff8f00u);
786 ASSERT_EQ(0xd000u, GetOutputInsn16(thunk_offset + 4u) & 0xff00u); // BEQ
787 gray_check_offset += 6u;
788 }
789 }
790 // Verify that the lock word for gray bit check is loaded from the holder address.
791 ASSERT_GE(output_.size() - gray_check_offset,
792 4u * /* 32-bit instructions */ 4u + 2u * /* 16-bit instructions */ 2u);
793 const uint32_t load_lock_word =
794 kLdrWInsn |
795 (holder_reg << 16) |
796 (/* IP */ 12 << 12) |
797 mirror::Object::MonitorOffset().Uint32Value();
798 ASSERT_EQ(load_lock_word, GetOutputInsn32(gray_check_offset));
799 // Verify the gray bit check.
800 DCHECK_GE(LockWord::kReadBarrierStateShift, 8u); // ROR modified immediate.
801 uint32_t ror_shift = 7 + (32 - LockWord::kReadBarrierStateShift);
802 const uint32_t tst_gray_bit_without_offset =
803 0xf0100f00 | (/* IP */ 12 << 16)
804 | (((ror_shift >> 4) & 1) << 26) // i
805 | (((ror_shift >> 1) & 7) << 12) // imm3
806 | ((ror_shift & 1) << 7); // imm8, ROR('1':imm8<7:0>, ror_shift).
807 EXPECT_EQ(tst_gray_bit_without_offset, GetOutputInsn32(gray_check_offset + 4u));
808 EXPECT_EQ(0xd100u, GetOutputInsn16(gray_check_offset + 8u) & 0xff00u); // BNE
809 // Verify the fake dependency (skip "ADD LR, LR, #ldr_offset").
810 const uint32_t fake_dependency =
811 0xeb000010 | // ADD Rd, Rn, Rm, LSR 32 (type=01, imm3=000, imm2=00)
812 (/* IP */ 12) | // Rm = IP
813 (base_reg << 16) | // Rn = base_reg
814 (base_reg << 8); // Rd = base_reg
815 EXPECT_EQ(fake_dependency, GetOutputInsn32(gray_check_offset + 14u));
816 // Do not check the rest of the implementation.
817
818 // The next thunk follows on the next aligned offset.
819 thunk_offset += RoundUp(expected_thunk.size(), kArmCodeAlignment);
820 }
821 }
822 }
823
TestBakerFieldNarrow(uint32_t offset,uint32_t ref_reg,bool implicit_null_checks)824 void Thumb2RelativePatcherTest::TestBakerFieldNarrow(uint32_t offset,
825 uint32_t ref_reg,
826 bool implicit_null_checks) {
827 implicit_null_checks_ = implicit_null_checks;
828 DCHECK_ALIGNED(offset, 4u);
829 DCHECK_LT(offset, 32u);
830 constexpr size_t kMethodCodeSize = 6u;
831 constexpr size_t kLiteralOffset = 0u;
832 uint32_t method_idx = 0u;
833 for (uint32_t base_reg : kBakerValidRegs) {
834 if (base_reg >= 8u) {
835 continue;
836 }
837 for (uint32_t holder_reg : kBakerValidRegs) {
838 uint32_t ldr = kLdrInsn | (offset << (6 - 2)) | (base_reg << 3) | ref_reg;
839 const std::vector<uint8_t> raw_code = RawCode({kBneWPlus0, ldr});
840 ASSERT_EQ(kMethodCodeSize, raw_code.size());
841 ArrayRef<const uint8_t> code(raw_code);
842 uint32_t encoded_data = EncodeBakerReadBarrierFieldData(
843 base_reg, holder_reg, /* narrow */ true);
844 const LinkerPatch patches[] = {
845 LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset, encoded_data),
846 };
847 ++method_idx;
848 AddCompiledMethod(MethodRef(method_idx), code, ArrayRef<const LinkerPatch>(patches));
849 }
850 }
851 Link();
852
853 // All thunks are at the end.
854 uint32_t thunk_offset = GetMethodOffset(method_idx) + RoundUp(kMethodCodeSize, kArmCodeAlignment);
855 method_idx = 0u;
856 for (uint32_t base_reg : kBakerValidRegs) {
857 if (base_reg >= 8u) {
858 continue;
859 }
860 for (uint32_t holder_reg : kBakerValidRegs) {
861 ++method_idx;
862 uint32_t bne = BneWWithOffset(GetMethodOffset(method_idx) + kLiteralOffset, thunk_offset);
863 uint32_t ldr = kLdrInsn | (offset << (6 - 2)) | (base_reg << 3) | ref_reg;
864 const std::vector<uint8_t> expected_code = RawCode({bne, ldr});
865 ASSERT_EQ(kMethodCodeSize, expected_code.size()) << "bne=0x" << std::hex << bne;
866 ASSERT_TRUE(
867 CheckLinkedMethod(MethodRef(method_idx), ArrayRef<const uint8_t>(expected_code)));
868
869 std::vector<uint8_t> expected_thunk =
870 CompileBakerOffsetThunk(base_reg, holder_reg, /* narrow */ true);
871 ASSERT_GT(output_.size(), thunk_offset);
872 ASSERT_GE(output_.size() - thunk_offset, expected_thunk.size());
873 ArrayRef<const uint8_t> compiled_thunk(output_.data() + thunk_offset,
874 expected_thunk.size());
875 if (ArrayRef<const uint8_t>(expected_thunk) != compiled_thunk) {
876 DumpDiff(ArrayRef<const uint8_t>(expected_thunk), compiled_thunk);
877 ASSERT_TRUE(false);
878 }
879
880 size_t gray_check_offset = thunk_offset;
881 if (implicit_null_checks && holder_reg == base_reg) {
882 // Verify that the null-check uses the correct register, i.e. holder_reg.
883 if (holder_reg < 8) {
884 ASSERT_GE(output_.size() - gray_check_offset, 2u);
885 ASSERT_EQ(0xb100 | holder_reg, GetOutputInsn16(thunk_offset) & 0xfd07u);
886 gray_check_offset +=2u;
887 } else {
888 ASSERT_GE(output_.size() - gray_check_offset, 6u);
889 ASSERT_EQ(0xf1b00f00u | (holder_reg << 16), GetOutputInsn32(thunk_offset) & 0xfbff8f00u);
890 ASSERT_EQ(0xd000u, GetOutputInsn16(thunk_offset + 4u) & 0xff00u); // BEQ
891 gray_check_offset += 6u;
892 }
893 }
894 // Verify that the lock word for gray bit check is loaded from the holder address.
895 ASSERT_GE(output_.size() - gray_check_offset,
896 4u * /* 32-bit instructions */ 4u + 2u * /* 16-bit instructions */ 2u);
897 const uint32_t load_lock_word =
898 kLdrWInsn |
899 (holder_reg << 16) |
900 (/* IP */ 12 << 12) |
901 mirror::Object::MonitorOffset().Uint32Value();
902 ASSERT_EQ(load_lock_word, GetOutputInsn32(gray_check_offset));
903 // Verify the gray bit check.
904 DCHECK_GE(LockWord::kReadBarrierStateShift, 8u); // ROR modified immediate.
905 uint32_t ror_shift = 7 + (32 - LockWord::kReadBarrierStateShift);
906 const uint32_t tst_gray_bit_without_offset =
907 0xf0100f00 | (/* IP */ 12 << 16)
908 | (((ror_shift >> 4) & 1) << 26) // i
909 | (((ror_shift >> 1) & 7) << 12) // imm3
910 | ((ror_shift & 1) << 7); // imm8, ROR('1':imm8<7:0>, ror_shift).
911 EXPECT_EQ(tst_gray_bit_without_offset, GetOutputInsn32(gray_check_offset + 4u));
912 EXPECT_EQ(0xd100u, GetOutputInsn16(gray_check_offset + 8u) & 0xff00u); // BNE
913 // Verify the fake dependency (skip "ADD LR, LR, #ldr_offset").
914 const uint32_t fake_dependency =
915 0xeb000010 | // ADD Rd, Rn, Rm, LSR 32 (type=01, imm3=000, imm2=00)
916 (/* IP */ 12) | // Rm = IP
917 (base_reg << 16) | // Rn = base_reg
918 (base_reg << 8); // Rd = base_reg
919 EXPECT_EQ(fake_dependency, GetOutputInsn32(gray_check_offset + 14u));
920 // Do not check the rest of the implementation.
921
922 // The next thunk follows on the next aligned offset.
923 thunk_offset += RoundUp(expected_thunk.size(), kArmCodeAlignment);
924 }
925 }
926 }
927
TEST_F(Thumb2RelativePatcherTest,BakerOffsetWide)928 TEST_F(Thumb2RelativePatcherTest, BakerOffsetWide) {
929 struct TestCase {
930 uint32_t offset;
931 uint32_t ref_reg;
932 };
933 static const TestCase test_cases[] = {
934 { 0u, 0u },
935 { 8u, 3u },
936 { 28u, 7u },
937 { 0xffcu, 11u },
938 };
939 for (const TestCase& test_case : test_cases) {
940 Reset();
941 TestBakerFieldWide(test_case.offset, test_case.ref_reg, /*implicit_null_checks=*/ true);
942 Reset();
943 TestBakerFieldWide(test_case.offset, test_case.ref_reg, /*implicit_null_checks=*/ false);
944 }
945 }
946
TEST_F(Thumb2RelativePatcherTest,BakerOffsetNarrow)947 TEST_F(Thumb2RelativePatcherTest, BakerOffsetNarrow) {
948 struct TestCase {
949 uint32_t offset;
950 uint32_t ref_reg;
951 };
952 static const TestCase test_cases[] = {
953 { 0, 0u },
954 { 8, 3u },
955 { 28, 7u },
956 };
957 for (const TestCase& test_case : test_cases) {
958 Reset();
959 TestBakerFieldNarrow(test_case.offset, test_case.ref_reg, /*implicit_null_checks=*/ true);
960 Reset();
961 TestBakerFieldNarrow(test_case.offset, test_case.ref_reg, /*implicit_null_checks=*/ false);
962 }
963 }
964
TEST_F(Thumb2RelativePatcherTest,BakerOffsetThunkInTheMiddle)965 TEST_F(Thumb2RelativePatcherTest, BakerOffsetThunkInTheMiddle) {
966 // One thunk in the middle with maximum distance branches to it from both sides.
967 // Use offset = 0, base_reg = 0, ref_reg = 0, the LDR is simply `kLdrWInsn`.
968 constexpr uint32_t kLiteralOffset1 = 6u;
969 const std::vector<uint8_t> raw_code1 = RawCode({kNopWInsn, kNopInsn, kBneWPlus0, kLdrWInsn});
970 ArrayRef<const uint8_t> code1(raw_code1);
971 uint32_t encoded_data = EncodeBakerReadBarrierFieldData(
972 /* base_reg */ 0, /* holder_reg */ 0, /* narrow */ false);
973 const LinkerPatch patches1[] = {
974 LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset1, encoded_data),
975 };
976 AddCompiledMethod(MethodRef(1u), code1, ArrayRef<const LinkerPatch>(patches1));
977
978 constexpr uint32_t expected_thunk_offset =
979 kLiteralOffset1 + kPcAdjustment + /* kMaxBcondPositiveDisplacement */ ((1 << 20) - 2u);
980 static_assert(IsAligned<kArmCodeAlignment>(expected_thunk_offset),
981 "Target offset must be aligned.");
982 size_t filler1_size = expected_thunk_offset -
983 RoundUp(raw_code1.size() + sizeof(OatQuickMethodHeader), kArmCodeAlignment);
984 std::vector<uint8_t> raw_filler1_code = GenNops(filler1_size / 2u);
985 ArrayRef<const uint8_t> filler1_code(raw_filler1_code);
986 AddCompiledMethod(MethodRef(2u), filler1_code);
987
988 // Enforce thunk reservation with a tiny method.
989 AddCompiledMethod(MethodRef(3u), kNopCode);
990
991 constexpr uint32_t kLiteralOffset2 = 4;
992 static_assert(IsAligned<kArmCodeAlignment>(kLiteralOffset2 + kPcAdjustment),
993 "PC for BNE must be aligned.");
994
995 // Allow reaching the thunk from the very beginning of a method almost 1MiB away. Backward branch
996 // reaches the full 1MiB but we need to take PC adjustment into account. Things to subtract:
997 // - thunk size and method 3 pre-header, rounded up (padding in between if needed)
998 // - method 3 code and method 4 pre-header, rounded up (padding in between if needed)
999 // - method 4 header (let there be no padding between method 4 code and method 5 pre-header).
1000 size_t thunk_size =
1001 CompileBakerOffsetThunk(/* base_reg */ 0, /* holder_reg */ 0, /* narrow */ false).size();
1002 size_t filler2_size =
1003 1 * MB - (kLiteralOffset2 + kPcAdjustment)
1004 - RoundUp(thunk_size + sizeof(OatQuickMethodHeader), kArmCodeAlignment)
1005 - RoundUp(kNopCode.size() + sizeof(OatQuickMethodHeader), kArmCodeAlignment)
1006 - sizeof(OatQuickMethodHeader);
1007 std::vector<uint8_t> raw_filler2_code = GenNops(filler2_size / 2u);
1008 ArrayRef<const uint8_t> filler2_code(raw_filler2_code);
1009 AddCompiledMethod(MethodRef(4u), filler2_code);
1010
1011 const std::vector<uint8_t> raw_code2 = RawCode({kNopWInsn, kBneWPlus0, kLdrWInsn});
1012 ArrayRef<const uint8_t> code2(raw_code2);
1013 const LinkerPatch patches2[] = {
1014 LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset2, encoded_data),
1015 };
1016 AddCompiledMethod(MethodRef(5u), code2, ArrayRef<const LinkerPatch>(patches2));
1017
1018 Link();
1019
1020 uint32_t first_method_offset = GetMethodOffset(1u);
1021 uint32_t last_method_offset = GetMethodOffset(5u);
1022 EXPECT_EQ(2 * MB, last_method_offset - first_method_offset);
1023
1024 const uint32_t bne_max_forward = kBneWPlus0 | 0x003f2fff;
1025 const uint32_t bne_max_backward = kBneWPlus0 | 0x04000000;
1026 const std::vector<uint8_t> expected_code1 =
1027 RawCode({kNopWInsn, kNopInsn, bne_max_forward, kLdrWInsn});
1028 const std::vector<uint8_t> expected_code2 = RawCode({kNopWInsn, bne_max_backward, kLdrWInsn});
1029 ASSERT_TRUE(CheckLinkedMethod(MethodRef(1), ArrayRef<const uint8_t>(expected_code1)));
1030 ASSERT_TRUE(CheckLinkedMethod(MethodRef(5), ArrayRef<const uint8_t>(expected_code2)));
1031 }
1032
TEST_F(Thumb2RelativePatcherTest,BakerOffsetThunkBeforeFiller)1033 TEST_F(Thumb2RelativePatcherTest, BakerOffsetThunkBeforeFiller) {
1034 // Based on the first part of BakerOffsetThunkInTheMiddle but the BNE is one instruction
1035 // earlier, so the thunk is emitted before the filler.
1036 // Use offset = 0, base_reg = 0, ref_reg = 0, the LDR is simply `kLdrWInsn`.
1037 constexpr uint32_t kLiteralOffset1 = 4u;
1038 const std::vector<uint8_t> raw_code1 = RawCode({kNopWInsn, kBneWPlus0, kLdrWInsn, kNopInsn});
1039 ArrayRef<const uint8_t> code1(raw_code1);
1040 uint32_t encoded_data = EncodeBakerReadBarrierFieldData(
1041 /* base_reg */ 0, /* holder_reg */ 0, /* narrow */ false);
1042 const LinkerPatch patches1[] = {
1043 LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset1, encoded_data),
1044 };
1045 AddCompiledMethod(MethodRef(1u), code1, ArrayRef<const LinkerPatch>(patches1));
1046
1047 constexpr uint32_t expected_thunk_offset =
1048 kLiteralOffset1 + kPcAdjustment + /* kMaxBcondPositiveDisplacement + 2 */ (1u << 20);
1049 static_assert(IsAligned<kArmCodeAlignment>(expected_thunk_offset),
1050 "Target offset must be aligned.");
1051 size_t filler1_size = expected_thunk_offset -
1052 RoundUp(raw_code1.size() + sizeof(OatQuickMethodHeader), kArmCodeAlignment);
1053 std::vector<uint8_t> raw_filler1_code = GenNops(filler1_size / 2u);
1054 ArrayRef<const uint8_t> filler1_code(raw_filler1_code);
1055 AddCompiledMethod(MethodRef(2u), filler1_code);
1056
1057 Link();
1058
1059 const uint32_t bne =
1060 BneWWithOffset(kLiteralOffset1, RoundUp(raw_code1.size(), kArmCodeAlignment));
1061 const std::vector<uint8_t> expected_code1 = RawCode({kNopWInsn, bne, kLdrWInsn, kNopInsn});
1062 ASSERT_TRUE(CheckLinkedMethod(MethodRef(1), ArrayRef<const uint8_t>(expected_code1)));
1063 }
1064
TEST_F(Thumb2RelativePatcherTest,BakerOffsetThunkInTheMiddleUnreachableFromLast)1065 TEST_F(Thumb2RelativePatcherTest, BakerOffsetThunkInTheMiddleUnreachableFromLast) {
1066 // Based on the BakerOffsetThunkInTheMiddle but the BNE in the last method is preceded
1067 // by NOP and cannot reach the thunk in the middle, so we emit an extra thunk at the end.
1068 // Use offset = 0, base_reg = 0, ref_reg = 0, the LDR is simply `kLdrWInsn`.
1069 constexpr uint32_t kLiteralOffset1 = 6u;
1070 const std::vector<uint8_t> raw_code1 = RawCode({kNopWInsn, kNopInsn, kBneWPlus0, kLdrWInsn});
1071 ArrayRef<const uint8_t> code1(raw_code1);
1072 uint32_t encoded_data = EncodeBakerReadBarrierFieldData(
1073 /* base_reg */ 0, /* holder_reg */ 0, /* narrow */ false);
1074 const LinkerPatch patches1[] = {
1075 LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset1, encoded_data),
1076 };
1077 AddCompiledMethod(MethodRef(1u), code1, ArrayRef<const LinkerPatch>(patches1));
1078
1079 constexpr uint32_t expected_thunk_offset =
1080 kLiteralOffset1 + kPcAdjustment + /* kMaxBcondPositiveDisplacement */ ((1 << 20) - 2u);
1081 static_assert(IsAligned<kArmCodeAlignment>(expected_thunk_offset),
1082 "Target offset must be aligned.");
1083 size_t filler1_size = expected_thunk_offset -
1084 RoundUp(raw_code1.size() + sizeof(OatQuickMethodHeader), kArmCodeAlignment);
1085 std::vector<uint8_t> raw_filler1_code = GenNops(filler1_size / 2u);
1086 ArrayRef<const uint8_t> filler1_code(raw_filler1_code);
1087 AddCompiledMethod(MethodRef(2u), filler1_code);
1088
1089 // Enforce thunk reservation with a tiny method.
1090 AddCompiledMethod(MethodRef(3u), kNopCode);
1091
1092 constexpr uint32_t kReachableFromOffset2 = 4;
1093 constexpr uint32_t kLiteralOffset2 = kReachableFromOffset2 + 2;
1094 static_assert(IsAligned<kArmCodeAlignment>(kReachableFromOffset2 + kPcAdjustment),
1095 "PC for BNE must be aligned.");
1096
1097 // If not for the extra NOP, this would allow reaching the thunk from the BNE
1098 // of a method 1MiB away. Backward branch reaches the full 1MiB but we need to take
1099 // PC adjustment into account. Things to subtract:
1100 // - thunk size and method 3 pre-header, rounded up (padding in between if needed)
1101 // - method 3 code and method 4 pre-header, rounded up (padding in between if needed)
1102 // - method 4 header (let there be no padding between method 4 code and method 5 pre-header).
1103 size_t thunk_size =
1104 CompileBakerOffsetThunk(/* base_reg */ 0, /* holder_reg */ 0, /* narrow */ false).size();
1105 size_t filler2_size =
1106 1 * MB - (kReachableFromOffset2 + kPcAdjustment)
1107 - RoundUp(thunk_size + sizeof(OatQuickMethodHeader), kArmCodeAlignment)
1108 - RoundUp(kNopCode.size() + sizeof(OatQuickMethodHeader), kArmCodeAlignment)
1109 - sizeof(OatQuickMethodHeader);
1110 std::vector<uint8_t> raw_filler2_code = GenNops(filler2_size / 2u);
1111 ArrayRef<const uint8_t> filler2_code(raw_filler2_code);
1112 AddCompiledMethod(MethodRef(4u), filler2_code);
1113
1114 // Extra 16-bit NOP compared to BakerOffsetThunkInTheMiddle.
1115 const std::vector<uint8_t> raw_code2 = RawCode({kNopWInsn, kNopInsn, kBneWPlus0, kLdrWInsn});
1116 ArrayRef<const uint8_t> code2(raw_code2);
1117 const LinkerPatch patches2[] = {
1118 LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset2, encoded_data),
1119 };
1120 AddCompiledMethod(MethodRef(5u), code2, ArrayRef<const LinkerPatch>(patches2));
1121
1122 Link();
1123
1124 uint32_t first_method_offset = GetMethodOffset(1u);
1125 uint32_t last_method_offset = GetMethodOffset(5u);
1126 EXPECT_EQ(2 * MB, last_method_offset - first_method_offset);
1127
1128 const uint32_t bne_max_forward = kBneWPlus0 | 0x003f2fff;
1129 const uint32_t bne_last =
1130 BneWWithOffset(kLiteralOffset2, RoundUp(raw_code2.size(), kArmCodeAlignment));
1131 const std::vector<uint8_t> expected_code1 =
1132 RawCode({kNopWInsn, kNopInsn, bne_max_forward, kLdrWInsn});
1133 const std::vector<uint8_t> expected_code2 =
1134 RawCode({kNopWInsn, kNopInsn, bne_last, kLdrWInsn});
1135 ASSERT_TRUE(CheckLinkedMethod(MethodRef(1), ArrayRef<const uint8_t>(expected_code1)));
1136 ASSERT_TRUE(CheckLinkedMethod(MethodRef(5), ArrayRef<const uint8_t>(expected_code2)));
1137 }
1138
TEST_F(Thumb2RelativePatcherTest,BakerArray)1139 TEST_F(Thumb2RelativePatcherTest, BakerArray) {
1140 auto ldr = [](uint32_t base_reg) {
1141 uint32_t index_reg = (base_reg == 0u) ? 1u : 0u;
1142 uint32_t ref_reg = (base_reg == 2) ? 3u : 2u;
1143 return kLdrRegLsl2 | index_reg | (base_reg << 16) | (ref_reg << 12);
1144 };
1145 constexpr size_t kMethodCodeSize = 8u;
1146 constexpr size_t kLiteralOffset = 0u;
1147 uint32_t method_idx = 0u;
1148 for (uint32_t base_reg : kBakerValidRegs) {
1149 ++method_idx;
1150 const std::vector<uint8_t> raw_code = RawCode({kBneWPlus0, ldr(base_reg)});
1151 ASSERT_EQ(kMethodCodeSize, raw_code.size());
1152 ArrayRef<const uint8_t> code(raw_code);
1153 const LinkerPatch patches[] = {
1154 LinkerPatch::BakerReadBarrierBranchPatch(
1155 kLiteralOffset, EncodeBakerReadBarrierArrayData(base_reg)),
1156 };
1157 AddCompiledMethod(MethodRef(method_idx), code, ArrayRef<const LinkerPatch>(patches));
1158 }
1159 Link();
1160
1161 // All thunks are at the end.
1162 uint32_t thunk_offset = GetMethodOffset(method_idx) + RoundUp(kMethodCodeSize, kArmCodeAlignment);
1163 method_idx = 0u;
1164 for (uint32_t base_reg : kBakerValidRegs) {
1165 ++method_idx;
1166 uint32_t bne = BneWWithOffset(GetMethodOffset(method_idx) + kLiteralOffset, thunk_offset);
1167 const std::vector<uint8_t> expected_code = RawCode({bne, ldr(base_reg)});
1168 ASSERT_EQ(kMethodCodeSize, expected_code.size());
1169 EXPECT_TRUE(CheckLinkedMethod(MethodRef(method_idx), ArrayRef<const uint8_t>(expected_code)));
1170
1171 std::vector<uint8_t> expected_thunk = CompileBakerArrayThunk(base_reg);
1172 ASSERT_GT(output_.size(), thunk_offset);
1173 ASSERT_GE(output_.size() - thunk_offset, expected_thunk.size());
1174 ArrayRef<const uint8_t> compiled_thunk(output_.data() + thunk_offset,
1175 expected_thunk.size());
1176 if (ArrayRef<const uint8_t>(expected_thunk) != compiled_thunk) {
1177 DumpDiff(ArrayRef<const uint8_t>(expected_thunk), compiled_thunk);
1178 ASSERT_TRUE(false);
1179 }
1180
1181 // Verify that the lock word for gray bit check is loaded from the correct address
1182 // before the base_reg which points to the array data.
1183 ASSERT_GE(output_.size() - thunk_offset,
1184 4u * /* 32-bit instructions */ 4u + 2u * /* 16-bit instructions */ 2u);
1185 int32_t data_offset =
1186 mirror::Array::DataOffset(Primitive::ComponentSize(Primitive::kPrimNot)).Int32Value();
1187 int32_t offset = mirror::Object::MonitorOffset().Int32Value() - data_offset;
1188 ASSERT_LT(offset, 0);
1189 ASSERT_GT(offset, -256);
1190 const uint32_t load_lock_word =
1191 kLdrNegativeOffset |
1192 (-offset & 0xffu) |
1193 (base_reg << 16) |
1194 (/* IP */ 12 << 12);
1195 EXPECT_EQ(load_lock_word, GetOutputInsn32(thunk_offset));
1196 // Verify the gray bit check.
1197 DCHECK_GE(LockWord::kReadBarrierStateShift, 8u); // ROR modified immediate.
1198 uint32_t ror_shift = 7 + (32 - LockWord::kReadBarrierStateShift);
1199 const uint32_t tst_gray_bit_without_offset =
1200 0xf0100f00 | (/* IP */ 12 << 16)
1201 | (((ror_shift >> 4) & 1) << 26) // i
1202 | (((ror_shift >> 1) & 7) << 12) // imm3
1203 | ((ror_shift & 1) << 7); // imm8, ROR('1':imm8<7:0>, ror_shift).
1204 EXPECT_EQ(tst_gray_bit_without_offset, GetOutputInsn32(thunk_offset + 4u));
1205 EXPECT_EQ(0xd100u, GetOutputInsn16(thunk_offset + 8u) & 0xff00u); // BNE
1206 // Verify the fake dependency.
1207 const uint32_t fake_dependency =
1208 0xeb000010 | // ADD Rd, Rn, Rm, LSR 32 (type=01, imm3=000, imm2=00)
1209 (/* IP */ 12) | // Rm = IP
1210 (base_reg << 16) | // Rn = base_reg
1211 (base_reg << 8); // Rd = base_reg
1212 EXPECT_EQ(fake_dependency, GetOutputInsn32(thunk_offset + 14u));
1213 // Do not check the rest of the implementation.
1214
1215 // The next thunk follows on the next aligned offset.
1216 thunk_offset += RoundUp(expected_thunk.size(), kArmCodeAlignment);
1217 }
1218 }
1219
TEST_F(Thumb2RelativePatcherTest,BakerGcRootWide)1220 TEST_F(Thumb2RelativePatcherTest, BakerGcRootWide) {
1221 constexpr size_t kMethodCodeSize = 8u;
1222 constexpr size_t kLiteralOffset = 4u;
1223 uint32_t method_idx = 0u;
1224 for (uint32_t root_reg : kBakerValidRegs) {
1225 ++method_idx;
1226 uint32_t ldr = kLdrWInsn | (/* offset */ 8) | (/* base_reg */ 0 << 16) | (root_reg << 12);
1227 const std::vector<uint8_t> raw_code = RawCode({ldr, kBneWPlus0});
1228 ASSERT_EQ(kMethodCodeSize, raw_code.size());
1229 ArrayRef<const uint8_t> code(raw_code);
1230 const LinkerPatch patches[] = {
1231 LinkerPatch::BakerReadBarrierBranchPatch(
1232 kLiteralOffset, EncodeBakerReadBarrierGcRootData(root_reg, /* narrow */ false)),
1233 };
1234 AddCompiledMethod(MethodRef(method_idx), code, ArrayRef<const LinkerPatch>(patches));
1235 }
1236 Link();
1237
1238 // All thunks are at the end.
1239 uint32_t thunk_offset = GetMethodOffset(method_idx) + RoundUp(kMethodCodeSize, kArmCodeAlignment);
1240 method_idx = 0u;
1241 for (uint32_t root_reg : kBakerValidRegs) {
1242 ++method_idx;
1243 uint32_t bne = BneWWithOffset(GetMethodOffset(method_idx) + kLiteralOffset, thunk_offset);
1244 uint32_t ldr = kLdrWInsn | (/* offset */ 8) | (/* base_reg */ 0 << 16) | (root_reg << 12);
1245 const std::vector<uint8_t> expected_code = RawCode({ldr, bne});
1246 ASSERT_EQ(kMethodCodeSize, expected_code.size());
1247 EXPECT_TRUE(CheckLinkedMethod(MethodRef(method_idx), ArrayRef<const uint8_t>(expected_code)));
1248
1249 std::vector<uint8_t> expected_thunk = CompileBakerGcRootThunk(root_reg, /* narrow */ false);
1250 ASSERT_GT(output_.size(), thunk_offset);
1251 ASSERT_GE(output_.size() - thunk_offset, expected_thunk.size());
1252 ArrayRef<const uint8_t> compiled_thunk(output_.data() + thunk_offset,
1253 expected_thunk.size());
1254 if (ArrayRef<const uint8_t>(expected_thunk) != compiled_thunk) {
1255 DumpDiff(ArrayRef<const uint8_t>(expected_thunk), compiled_thunk);
1256 ASSERT_TRUE(false);
1257 }
1258
1259 // Verify that the fast-path null-check uses the correct register, i.e. root_reg.
1260 if (root_reg < 8) {
1261 ASSERT_GE(output_.size() - thunk_offset, 2u);
1262 ASSERT_EQ(0xb100 | root_reg, GetOutputInsn16(thunk_offset) & 0xfd07u);
1263 } else {
1264 ASSERT_GE(output_.size() - thunk_offset, 6u);
1265 ASSERT_EQ(0xf1b00f00u | (root_reg << 16), GetOutputInsn32(thunk_offset) & 0xfbff8f00u);
1266 ASSERT_EQ(0xd000u, GetOutputInsn16(thunk_offset + 4u) & 0xff00u); // BEQ
1267 }
1268 // Do not check the rest of the implementation.
1269
1270 // The next thunk follows on the next aligned offset.
1271 thunk_offset += RoundUp(expected_thunk.size(), kArmCodeAlignment);
1272 }
1273 }
1274
TEST_F(Thumb2RelativePatcherTest,BakerGcRootNarrow)1275 TEST_F(Thumb2RelativePatcherTest, BakerGcRootNarrow) {
1276 constexpr size_t kMethodCodeSize = 6u;
1277 constexpr size_t kLiteralOffset = 2u;
1278 uint32_t method_idx = 0u;
1279 for (uint32_t root_reg : kBakerValidRegsNarrow) {
1280 ++method_idx;
1281 uint32_t ldr = kLdrInsn | (/* offset */ 8 << (6 - 2)) | (/* base_reg */ 0 << 3) | root_reg;
1282 const std::vector<uint8_t> raw_code = RawCode({ldr, kBneWPlus0});
1283 ASSERT_EQ(kMethodCodeSize, raw_code.size());
1284 ArrayRef<const uint8_t> code(raw_code);
1285 const LinkerPatch patches[] = {
1286 LinkerPatch::BakerReadBarrierBranchPatch(
1287 kLiteralOffset, EncodeBakerReadBarrierGcRootData(root_reg, /* narrow */ true)),
1288 };
1289 AddCompiledMethod(MethodRef(method_idx), code, ArrayRef<const LinkerPatch>(patches));
1290 }
1291 Link();
1292
1293 // All thunks are at the end.
1294 uint32_t thunk_offset = GetMethodOffset(method_idx) + RoundUp(kMethodCodeSize, kArmCodeAlignment);
1295 method_idx = 0u;
1296 for (uint32_t root_reg : kBakerValidRegsNarrow) {
1297 ++method_idx;
1298 uint32_t bne = BneWWithOffset(GetMethodOffset(method_idx) + kLiteralOffset, thunk_offset);
1299 uint32_t ldr = kLdrInsn | (/* offset */ 8 << (6 - 2)) | (/* base_reg */ 0 << 3) | root_reg;
1300 const std::vector<uint8_t> expected_code = RawCode({ldr, bne});
1301 ASSERT_EQ(kMethodCodeSize, expected_code.size());
1302 EXPECT_TRUE(CheckLinkedMethod(MethodRef(method_idx), ArrayRef<const uint8_t>(expected_code)));
1303
1304 std::vector<uint8_t> expected_thunk = CompileBakerGcRootThunk(root_reg, /* narrow */ true);
1305 ASSERT_GT(output_.size(), thunk_offset);
1306 ASSERT_GE(output_.size() - thunk_offset, expected_thunk.size());
1307 ArrayRef<const uint8_t> compiled_thunk(output_.data() + thunk_offset,
1308 expected_thunk.size());
1309 if (ArrayRef<const uint8_t>(expected_thunk) != compiled_thunk) {
1310 DumpDiff(ArrayRef<const uint8_t>(expected_thunk), compiled_thunk);
1311 ASSERT_TRUE(false);
1312 }
1313
1314 // Verify that the fast-path null-check CBZ uses the correct register, i.e. root_reg.
1315 ASSERT_GE(output_.size() - thunk_offset, 2u);
1316 ASSERT_EQ(0xb100 | root_reg, GetOutputInsn16(thunk_offset) & 0xfd07u);
1317 // Do not check the rest of the implementation.
1318
1319 // The next thunk follows on the next aligned offset.
1320 thunk_offset += RoundUp(expected_thunk.size(), kArmCodeAlignment);
1321 }
1322 }
1323
TEST_F(Thumb2RelativePatcherTest,BakerGcRootOffsetBits)1324 TEST_F(Thumb2RelativePatcherTest, BakerGcRootOffsetBits) {
1325 // Test 1MiB of patches to the same thunk to stress-test different large offsets.
1326 // (The low bits are not that important but the location of the high bits is easy to get wrong.)
1327 std::vector<uint8_t> code;
1328 code.reserve(1 * MB);
1329 const size_t num_patches = 1 * MB / 8u;
1330 std::vector<LinkerPatch> patches;
1331 patches.reserve(num_patches);
1332 const uint32_t ldr =
1333 kLdrWInsn | (/* offset */ 8) | (/* base_reg */ 0 << 16) | (/* root_reg */ 0 << 12);
1334 uint32_t encoded_data = EncodeBakerReadBarrierGcRootData(/* root_reg */ 0, /* narrow */ false);
1335 for (size_t i = 0; i != num_patches; ++i) {
1336 PushBackInsn(&code, ldr);
1337 PushBackInsn(&code, kBneWPlus0);
1338 patches.push_back(LinkerPatch::BakerReadBarrierBranchPatch(8u * i + 4u, encoded_data));
1339 }
1340 ASSERT_EQ(1 * MB, code.size());
1341 ASSERT_EQ(num_patches, patches.size());
1342 AddCompiledMethod(MethodRef(1u),
1343 ArrayRef<const uint8_t>(code),
1344 ArrayRef<const LinkerPatch>(patches));
1345 Link();
1346
1347 // The thunk is right after the method code.
1348 DCHECK_ALIGNED(1 * MB, kArmCodeAlignment);
1349 std::vector<uint8_t> expected_code;
1350 for (size_t i = 0; i != num_patches; ++i) {
1351 PushBackInsn(&expected_code, ldr);
1352 PushBackInsn(&expected_code, BneWWithOffset(8u * i + 4u, 1 * MB));
1353 patches.push_back(LinkerPatch::BakerReadBarrierBranchPatch(8u * i + 4u, encoded_data));
1354 }
1355 EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(expected_code)));
1356 }
1357
TEST_F(Thumb2RelativePatcherTest,BakerAndMethodCallInteraction)1358 TEST_F(Thumb2RelativePatcherTest, BakerAndMethodCallInteraction) {
1359 // During development, there was a `DCHECK_LE(MaxNextOffset(), next_thunk.MaxNextOffset());`
1360 // in `ArmBaseRelativePatcher::ThunkData::MakeSpaceBefore()` which does not necessarily
1361 // hold when we're reserving thunks of different sizes. This test exposes the situation
1362 // by using Baker thunks and a method call thunk.
1363
1364 // Add a method call patch that can reach to method 1 offset + 16MiB.
1365 uint32_t method_idx = 0u;
1366 constexpr size_t kMethodCallLiteralOffset = 2u;
1367 constexpr uint32_t kMissingMethodIdx = 2u;
1368 const std::vector<uint8_t> raw_code1 = RawCode({kNopInsn, kBlPlus0});
1369 const LinkerPatch method1_patches[] = {
1370 LinkerPatch::RelativeCodePatch(kMethodCallLiteralOffset, nullptr, 2u),
1371 };
1372 ArrayRef<const uint8_t> code1(raw_code1);
1373 ++method_idx;
1374 AddCompiledMethod(MethodRef(1u), code1, ArrayRef<const LinkerPatch>(method1_patches));
1375
1376 // Skip kMissingMethodIdx.
1377 ++method_idx;
1378 ASSERT_EQ(kMissingMethodIdx, method_idx);
1379 // Add a method with the right size that the method code for the next one starts 1MiB
1380 // after code for method 1.
1381 size_t filler_size =
1382 1 * MB - RoundUp(raw_code1.size() + sizeof(OatQuickMethodHeader), kArmCodeAlignment)
1383 - sizeof(OatQuickMethodHeader);
1384 std::vector<uint8_t> filler_code = GenNops(filler_size / 2u);
1385 ++method_idx;
1386 AddCompiledMethod(MethodRef(method_idx), ArrayRef<const uint8_t>(filler_code));
1387 // Add 14 methods with 1MiB code+header, making the code for the next method start 1MiB
1388 // before the currently scheduled MaxNextOffset() for the method call thunk.
1389 for (uint32_t i = 0; i != 14; ++i) {
1390 filler_size = 1 * MB - sizeof(OatQuickMethodHeader);
1391 filler_code = GenNops(filler_size / 2u);
1392 ++method_idx;
1393 AddCompiledMethod(MethodRef(method_idx), ArrayRef<const uint8_t>(filler_code));
1394 }
1395
1396 // Add 2 Baker GC root patches to the last method, one that would allow the thunk at
1397 // 1MiB + kArmCodeAlignment, i.e. kArmCodeAlignment after the method call thunk, and the
1398 // second that needs it kArmCodeAlignment after that. Given the size of the GC root thunk
1399 // is more than the space required by the method call thunk plus kArmCodeAlignment,
1400 // this pushes the first GC root thunk's pending MaxNextOffset() before the method call
1401 // thunk's pending MaxNextOffset() which needs to be adjusted.
1402 ASSERT_LT(RoundUp(CompileMethodCallThunk().size(), kArmCodeAlignment) + kArmCodeAlignment,
1403 CompileBakerGcRootThunk(/* root_reg */ 0, /* narrow */ false).size());
1404 static_assert(kArmCodeAlignment == 8, "Code below assumes kArmCodeAlignment == 8");
1405 constexpr size_t kBakerLiteralOffset1 = kArmCodeAlignment + 2u - kPcAdjustment;
1406 constexpr size_t kBakerLiteralOffset2 = kBakerLiteralOffset1 + kArmCodeAlignment;
1407 // Use offset = 0, base_reg = 0, the LDR is simply `kLdrWInsn | (root_reg << 12)`.
1408 const uint32_t ldr1 = kLdrWInsn | (/* root_reg */ 1 << 12);
1409 const uint32_t ldr2 = kLdrWInsn | (/* root_reg */ 2 << 12);
1410 const std::vector<uint8_t> last_method_raw_code = RawCode({
1411 kNopInsn, // Padding before first GC root read barrier.
1412 ldr1, kBneWPlus0, // First GC root LDR with read barrier.
1413 ldr2, kBneWPlus0, // Second GC root LDR with read barrier.
1414 });
1415 uint32_t encoded_data1 = EncodeBakerReadBarrierGcRootData(/* root_reg */ 1, /* narrow */ false);
1416 uint32_t encoded_data2 = EncodeBakerReadBarrierGcRootData(/* root_reg */ 2, /* narrow */ false);
1417 const LinkerPatch last_method_patches[] = {
1418 LinkerPatch::BakerReadBarrierBranchPatch(kBakerLiteralOffset1, encoded_data1),
1419 LinkerPatch::BakerReadBarrierBranchPatch(kBakerLiteralOffset2, encoded_data2),
1420 };
1421 ++method_idx;
1422 AddCompiledMethod(MethodRef(method_idx),
1423 ArrayRef<const uint8_t>(last_method_raw_code),
1424 ArrayRef<const LinkerPatch>(last_method_patches));
1425
1426 // The main purpose of the test is to check that Link() does not cause a crash.
1427 Link();
1428
1429 ASSERT_EQ(15 * MB, GetMethodOffset(method_idx) - GetMethodOffset(1u));
1430 }
1431
1432 } // namespace linker
1433 } // namespace art
1434