1 /*
2 * Copyright (C) 2021 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "berberis/base/large_mmap.h"
18
19 #include <atomic>
20
21 #include "berberis/base/mmap.h"
22
23 namespace berberis {
24
25 #if defined(__LP64__)
26
27 // Some apps have a bug of not supporting pointer difference >16gb. (http://b/167572400)
28 // Since translator allocates some additional address space it is more likely for app
29 // to get in the situation of having pointers more than 16g apart.
30 //
31 // The solution is to move large long-term translator allocations away from the current mmap
32 // area, leaving close addresses for the guest allocations, so that memory allocation footprint
33 // is closer to what app gets in the native environment.
34 //
35 // We implement this by allocating a huge buffer for translator allocations. To ensure enough space
36 // between the buffer and the current mmap area, we allocate an even larger buffer and then free a
37 // part of it. Since on modern Linux mmap moves top-to-down (https://lwn.net/Articles/91829) the
38 // spacing area (that needs to be freed) is at the higher addresses.
39 //
40 // ATTENTION: If guest allocation (in another thread) takes place while the buffer is allocated but
41 // the spacing is not yet freed, the allocation will go too far away from the current mmap area,
42 // manifesting the bug. To avoid that, we allocate the buffer on init and never reallocate it.
43 // When the buffer is exhausted, translator allocates with mmap directly.
44 //
45 // Note that we cannot simply allocate a huge buffer at init to achieve the same result for
46 // following reasons:
47 // 1. there can be small mapping gaps, that will later be taken by guest allocations.
48 // 2. Some mappings can be unmapped later also allowing new guest allocations in their place.
49
50 namespace {
51
52 // ATTENTION: buffer allocation is not atomic! To make it atomic, use PointerAndCounter!
53 std::atomic<uint8_t*> g_buffer = nullptr;
54 uint8_t* g_buffer_end = nullptr;
55
56 } // namespace
57
InitLargeMmap()58 void InitLargeMmap() {
59 constexpr size_t kBufferSize = size_t(1) << 34; // 16gb
60 constexpr size_t kSpacingSize = size_t(1) << 35; // 32gb
61
62 // As explained above we expect mmap to work top-to-down, so spacing is at the higher addresses.
63 auto* ptr = static_cast<uint8_t*>(MmapImplOrDie(
64 {.size = kBufferSize + kSpacingSize, .flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE}));
65 MunmapOrDie(ptr + kBufferSize, kSpacingSize);
66
67 g_buffer.store(ptr);
68 g_buffer_end = ptr + kBufferSize;
69 }
70
LargeMmapImplOrDie(MmapImplArgs args)71 void* LargeMmapImplOrDie(MmapImplArgs args) {
72 if (args.addr == nullptr) {
73 CHECK_EQ(0, args.flags & MAP_FIXED);
74 size_t size = AlignUpPageSize(args.size);
75
76 uint8_t* curr = g_buffer.load(std::memory_order_relaxed);
77 for (;;) {
78 uint8_t* next = curr + size;
79 if (next > g_buffer_end) {
80 break;
81 }
82
83 // Updates curr!
84 if (g_buffer.compare_exchange_weak(curr, next, std::memory_order_release)) {
85 args.addr = curr;
86 args.flags |= MAP_FIXED;
87 break;
88 }
89 }
90 }
91
92 return MmapImplOrDie(args);
93 }
94
95 #else
96
97 void InitLargeMmap() {}
98
99 void* LargeMmapImplOrDie(MmapImplArgs args) {
100 return MmapImplOrDie(args);
101 }
102
103 #endif
104
LargeMmapOrDie(size_t size)105 void* LargeMmapOrDie(size_t size) {
106 return LargeMmapImplOrDie({.size = size});
107 }
108
109 } // namespace berberis
110