1// Copyright (C) 2019 The Android Open Source Project
2// Copyright (C) 2019 Google Inc.
3//
4// Licensed under the Apache License, Version 2.0 (the "License");
5// you may not use this file except in compliance with the License.
6// You may obtain a copy of the License at
7//
8// http://www.apache.org/licenses/LICENSE-2.0
9//
10// Unless required by applicable law or agreed to in writing, software
11// distributed under the License is distributed on an "AS IS" BASIS,
12// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13// See the License for the specific language governing permissions and
14// limitations under the License.
15
16#include <linux/types.h>
17#include <linux/ioctl.h>
18#include <sys/types.h>
19#include <sys/stat.h>
20#include <sys/mman.h>
21#include <sys/ioctl.h>
22#include <fcntl.h>
23#include <unistd.h>
24#include <cstdlib>
25#include <errno.h>
26#include <memory>
27#include <cstring>
28
29#include "goldfish_address_space.h"
30#include <log/log.h>
31
32
33// See virgl_hw.h and p_defines.h
34#define VIRGL_FORMAT_R8_UNORM 64
35#define VIRGL_BIND_CUSTOM (1 << 17)
36#define PIPE_BUFFER 0
37
38#ifdef PAGE_SIZE
39constexpr size_t kPageSize = PAGE_SIZE;
40#else
41static const size_t kPageSize = getpagesize();
42#endif
43
44namespace {
45
46struct goldfish_address_space_allocate_block {
47    __u64 size;
48    __u64 offset;
49    __u64 phys_addr;
50};
51
52struct goldfish_address_space_claim_shared {
53    __u64 offset;
54    __u64 size;
55};
56
57#define GOLDFISH_ADDRESS_SPACE_IOCTL_MAGIC		'G'
58#define GOLDFISH_ADDRESS_SPACE_IOCTL_OP(OP, T)		_IOWR(GOLDFISH_ADDRESS_SPACE_IOCTL_MAGIC, OP, T)
59#define GOLDFISH_ADDRESS_SPACE_IOCTL_ALLOCATE_BLOCK	GOLDFISH_ADDRESS_SPACE_IOCTL_OP(10, struct goldfish_address_space_allocate_block)
60#define GOLDFISH_ADDRESS_SPACE_IOCTL_DEALLOCATE_BLOCK	GOLDFISH_ADDRESS_SPACE_IOCTL_OP(11, __u64)
61#define GOLDFISH_ADDRESS_SPACE_IOCTL_PING		GOLDFISH_ADDRESS_SPACE_IOCTL_OP(12, struct address_space_ping)
62#define GOLDFISH_ADDRESS_SPACE_IOCTL_CLAIM_SHARED		GOLDFISH_ADDRESS_SPACE_IOCTL_OP(13, struct goldfish_address_space_claim_shared)
63#define GOLDFISH_ADDRESS_SPACE_IOCTL_UNCLAIM_SHARED		GOLDFISH_ADDRESS_SPACE_IOCTL_OP(14, __u64)
64
65const char GOLDFISH_ADDRESS_SPACE_DEVICE_NAME[] = "/dev/goldfish_address_space";
66
67const int HOST_MEMORY_ALLOCATOR_COMMAND_ALLOCATE_ID = 1;
68const int HOST_MEMORY_ALLOCATOR_COMMAND_UNALLOCATE_ID = 2;
69
70int create_address_space_fd()
71{
72    return ::open(GOLDFISH_ADDRESS_SPACE_DEVICE_NAME, O_RDWR);
73}
74
75long ioctl_allocate(int fd, struct goldfish_address_space_allocate_block *request)
76{
77    return ::ioctl(fd, GOLDFISH_ADDRESS_SPACE_IOCTL_ALLOCATE_BLOCK, request);
78}
79
80long ioctl_deallocate(int fd, uint64_t offset)
81{
82    return ::ioctl(fd, GOLDFISH_ADDRESS_SPACE_IOCTL_DEALLOCATE_BLOCK, &offset);
83}
84
85long ioctl_ping(int fd, struct address_space_ping *request)
86{
87    return ::ioctl(fd, GOLDFISH_ADDRESS_SPACE_IOCTL_PING, request);
88}
89
90long set_address_space_subdevice_type(int fd, uint64_t type)
91{
92    struct address_space_ping request;
93    ::memset(&request, 0, sizeof(request));
94    request.resourceId = sizeof(request);
95    request.metadata = type;
96
97    long ret = ioctl_ping(fd, &request);
98    if (ret) {
99        return ret;
100    }
101
102    return request.metadata;
103}
104
105long ioctl_claim_shared(int fd, struct goldfish_address_space_claim_shared *request)
106{
107    return ::ioctl(fd, GOLDFISH_ADDRESS_SPACE_IOCTL_CLAIM_SHARED, request);
108}
109
110long ioctl_unclaim_shared(int fd, uint64_t offset)
111{
112    return ::ioctl(fd, GOLDFISH_ADDRESS_SPACE_IOCTL_UNCLAIM_SHARED, &offset);
113}
114
115}  // namespace
116
117GoldfishAddressSpaceBlockProvider::GoldfishAddressSpaceBlockProvider(GoldfishAddressSpaceSubdeviceType subdevice)
118  : m_handle(create_address_space_fd())
119{
120    if ((subdevice != GoldfishAddressSpaceSubdeviceType::NoSubdevice) && is_opened()) {
121        const long ret = set_address_space_subdevice_type(m_handle, subdevice);
122        if (ret != 0 && ret != subdevice) {  // TODO: retire the 'ret != subdevice' check
123            ALOGE("%s: set_address_space_subdevice_type failed for device_type=%lu, ret=%ld",
124                  __func__, static_cast<unsigned long>(subdevice), ret);
125            close();
126        }
127    }
128}
129
130GoldfishAddressSpaceBlockProvider::~GoldfishAddressSpaceBlockProvider()
131{
132    if (is_opened()) {
133        ::close(m_handle);
134    }
135}
136
137bool GoldfishAddressSpaceBlockProvider::is_opened() const
138{
139    return m_handle >= 0;
140}
141
142void GoldfishAddressSpaceBlockProvider::close()
143{
144    if (is_opened()) {
145        ::close(m_handle);
146        m_handle = -1;
147    }
148}
149
150address_space_handle_t GoldfishAddressSpaceBlockProvider::release()
151{
152    address_space_handle_t handle = m_handle;
153    m_handle = -1;
154    return handle;
155}
156
157void GoldfishAddressSpaceBlockProvider::closeHandle(address_space_handle_t handle)
158{
159    ::close(handle);
160}
161
162GoldfishAddressSpaceBlock::GoldfishAddressSpaceBlock()
163    : m_handle(-1)
164    , m_mmaped_ptr(NULL)
165    , m_phys_addr(0)
166    , m_host_addr(0)
167    , m_offset(0)
168    , m_size(0) {}
169
170GoldfishAddressSpaceBlock::~GoldfishAddressSpaceBlock()
171{
172    destroy();
173}
174
175GoldfishAddressSpaceBlock &GoldfishAddressSpaceBlock::operator=(const GoldfishAddressSpaceBlock &rhs)
176{
177    m_mmaped_ptr = rhs.m_mmaped_ptr;
178    m_phys_addr = rhs.m_phys_addr;
179    m_host_addr = rhs.m_host_addr;
180    m_offset = rhs.m_offset;
181    m_size = rhs.m_size;
182    m_handle = rhs.m_handle;
183
184    return *this;
185}
186
187bool GoldfishAddressSpaceBlock::allocate(GoldfishAddressSpaceBlockProvider *provider, size_t size)
188{
189    destroy();
190
191    if (!provider->is_opened()) {
192        return false;
193    }
194
195    struct goldfish_address_space_allocate_block request;
196    ::memset(&request, 0, sizeof(request));
197    request.size = size;
198
199    long res = ioctl_allocate(provider->m_handle, &request);
200    if (res) {
201        return false;
202    } else {
203        m_phys_addr = request.phys_addr;
204        m_offset = request.offset;
205        m_size = request.size;
206        m_handle = provider->m_handle;
207        m_is_shared_mapping = false;
208
209        return true;
210    }
211}
212
213bool GoldfishAddressSpaceBlock::claimShared(GoldfishAddressSpaceBlockProvider *provider, uint64_t offset, uint64_t size)
214{
215    destroy();
216
217    if (!provider->is_opened()) {
218        return false;
219    }
220
221    struct goldfish_address_space_claim_shared request;
222    request.offset = offset;
223    request.size = size;
224    long res = ioctl_claim_shared(provider->m_handle, &request);
225
226    if (res) {
227        return false;
228    }
229
230    m_offset = offset;
231    m_size = size;
232    m_handle = provider->m_handle;
233    m_is_shared_mapping = true;
234
235    return true;
236}
237
238uint64_t GoldfishAddressSpaceBlock::physAddr() const
239{
240    return m_phys_addr;
241}
242
243uint64_t GoldfishAddressSpaceBlock::hostAddr() const
244{
245    return m_host_addr;
246}
247
248void *GoldfishAddressSpaceBlock::mmap(uint64_t host_addr)
249{
250    if (m_size == 0) {
251        ALOGE("%s: called with zero size\n", __func__);
252        return NULL;
253    }
254    if (m_mmaped_ptr) {
255        ALOGE("'mmap' called for an already mmaped address block");
256        ::abort();
257    }
258
259    void *result;
260    const int res = memoryMap(NULL, m_size, m_handle, m_offset, &result);
261    if (res) {
262        ALOGE("%s: host memory map failed with size 0x%llx "
263              "off 0x%llx errno %d\n",
264              __func__,
265              (unsigned long long)m_size,
266              (unsigned long long)m_offset, res);
267        return NULL;
268    } else {
269        m_mmaped_ptr = result;
270        m_host_addr = host_addr;
271        return guestPtr();
272    }
273}
274
275void *GoldfishAddressSpaceBlock::guestPtr() const
276{
277    return reinterpret_cast<char *>(m_mmaped_ptr) + (m_host_addr & (kPageSize - 1));
278}
279
280void GoldfishAddressSpaceBlock::destroy()
281{
282    if (m_mmaped_ptr && m_size) {
283        memoryUnmap(m_mmaped_ptr, m_size);
284        m_mmaped_ptr = NULL;
285    }
286
287    if (m_size) {
288        long res = -EINVAL;
289
290        if (m_is_shared_mapping) {
291            res = ioctl_unclaim_shared(m_handle, m_offset);
292            if (res) {
293                ALOGE("ioctl_unclaim_shared failed, res=%ld", res);
294                ::abort();
295            }
296        } else {
297            res = ioctl_deallocate(m_handle, m_offset);
298            if (res) {
299                ALOGE("ioctl_deallocate failed, res=%ld", res);
300                ::abort();
301            }
302        }
303
304        m_is_shared_mapping = false;
305
306        m_phys_addr = 0;
307        m_host_addr = 0;
308        m_offset = 0;
309        m_size = 0;
310    }
311}
312
313void GoldfishAddressSpaceBlock::release()
314{
315    m_handle = -1;
316    m_mmaped_ptr = NULL;
317    m_phys_addr = 0;
318    m_host_addr = 0;
319    m_offset = 0;
320    m_size = 0;
321}
322
323int GoldfishAddressSpaceBlock::memoryMap(void *addr,
324                                         size_t len,
325                                         address_space_handle_t fd,
326                                         uint64_t off,
327                                         void** dst) {
328    void* ptr = ::mmap64(addr, len, PROT_WRITE, MAP_SHARED, fd, off);
329    if (MAP_FAILED == ptr) {
330        return errno;
331    } else {
332        *dst = ptr;
333        return 0;
334    }
335}
336
337void GoldfishAddressSpaceBlock::memoryUnmap(void *ptr, size_t size)
338{
339    ::munmap(ptr, size);
340}
341
342GoldfishAddressSpaceHostMemoryAllocator::GoldfishAddressSpaceHostMemoryAllocator(bool useSharedSlots)
343  : m_provider(useSharedSlots
344        ? GoldfishAddressSpaceSubdeviceType::SharedSlotsHostMemoryAllocator
345        : GoldfishAddressSpaceSubdeviceType::HostMemoryAllocator),
346    m_useSharedSlots(useSharedSlots)
347{}
348
349bool GoldfishAddressSpaceHostMemoryAllocator::is_opened() const { return m_provider.is_opened(); }
350
351long GoldfishAddressSpaceHostMemoryAllocator::hostMalloc(GoldfishAddressSpaceBlock *block, size_t size)
352{
353    if (size == 0) {
354        return -EINVAL;
355    }
356    if (block->size() > 0) {
357        return -EINVAL;
358    }
359    if (!m_provider.is_opened()) {
360        return -ENODEV;
361    }
362
363    struct address_space_ping request;
364    if (m_useSharedSlots) {
365        // shared memory slots are supported
366        ::memset(&request, 0, sizeof(request));
367        request.resourceId = sizeof(request);
368        request.size = size;
369        request.metadata = HOST_MEMORY_ALLOCATOR_COMMAND_ALLOCATE_ID;
370
371        long ret = ioctl_ping(m_provider.m_handle, &request);
372        if (ret) {
373            return ret;
374        }
375        ret = static_cast<long>(request.metadata);
376        if (ret) {
377            return ret;
378        }
379
380        block->claimShared(&m_provider, request.offset, request.size);
381    } else {
382        // shared memory slots are not supported
383        if (!block->allocate(&m_provider, size)) {
384            return -ENOMEM;
385        }
386
387        ::memset(&request, 0, sizeof(request));
388        request.resourceId = sizeof(request);
389        request.offset = block->offset();
390        request.size = block->size();
391        request.metadata = HOST_MEMORY_ALLOCATOR_COMMAND_ALLOCATE_ID;
392
393        long ret = ioctl_ping(m_provider.m_handle, &request);
394        if (ret) {
395            return ret;
396        }
397        ret = static_cast<long>(request.metadata);
398        if (ret) {
399            return ret;
400        }
401    }
402
403    block->mmap(0);
404    return 0;
405}
406
407void GoldfishAddressSpaceHostMemoryAllocator::hostFree(GoldfishAddressSpaceBlock *block)
408{
409    if (block->size() == 0) {
410        return;
411    }
412
413    if (!m_provider.is_opened()) {
414        ALOGE("%s: device is not available", __func__);
415        ::abort();
416    }
417
418    if (block->guestPtr()) {
419        struct address_space_ping request;
420        ::memset(&request, 0, sizeof(request));
421        request.resourceId = sizeof(request);
422        request.offset = block->offset();
423        request.metadata = HOST_MEMORY_ALLOCATOR_COMMAND_UNALLOCATE_ID;
424
425        const long ret = ioctl_ping(m_provider.m_handle, &request);
426        if (ret) {
427            ALOGE("%s: ioctl_ping failed, ret=%ld", __func__, ret);
428            ::abort();
429        }
430    }
431
432    block->replace(NULL);
433}
434
435address_space_handle_t goldfish_address_space_open() {
436    return ::open(GOLDFISH_ADDRESS_SPACE_DEVICE_NAME, O_RDWR);
437}
438
439void goldfish_address_space_close(address_space_handle_t handle) {
440    ::close(handle);
441}
442
443bool goldfish_address_space_allocate(
444    address_space_handle_t handle,
445    size_t size, uint64_t* phys_addr, uint64_t* offset) {
446
447    struct goldfish_address_space_allocate_block request;
448    ::memset(&request, 0, sizeof(request));
449    request.size = size;
450
451    long res = ioctl_allocate(handle, &request);
452
453    if (res) return false;
454
455    *phys_addr = request.phys_addr;
456    *offset = request.offset;
457    return true;
458}
459
460bool goldfish_address_space_free(
461    address_space_handle_t handle, uint64_t offset) {
462
463    long res = ioctl_deallocate(handle, offset);
464
465    if (res) {
466        ALOGE("ioctl_deallocate failed, res=%ld", res);
467        ::abort();
468    }
469
470    return true;
471}
472
473bool goldfish_address_space_claim_shared(
474    address_space_handle_t handle, uint64_t offset, uint64_t size) {
475
476    struct goldfish_address_space_claim_shared request;
477    request.offset = offset;
478    request.size = size;
479    long res = ioctl_claim_shared(handle, &request);
480
481    if (res) return false;
482
483    return true;
484}
485
486bool goldfish_address_space_unclaim_shared(
487        address_space_handle_t handle, uint64_t offset) {
488    long res = ioctl_unclaim_shared(handle, offset);
489    if (res) {
490        ALOGE("ioctl_unclaim_shared failed, res=%ld", res);
491        ::abort();
492    }
493
494    return true;
495}
496
497// pgoff is the offset into the page to return in the result
498void* goldfish_address_space_map(
499    address_space_handle_t handle,
500    uint64_t offset, uint64_t size,
501    uint64_t pgoff) {
502
503    void* res = ::mmap64(0, size, PROT_WRITE, MAP_SHARED, handle, offset);
504
505    if (res == MAP_FAILED) {
506        ALOGE("%s: failed to map. errno: %d\n", __func__, errno);
507        return 0;
508    }
509
510    return (void*)(((char*)res) + (uintptr_t)(pgoff & (kPageSize - 1)));
511}
512
513void goldfish_address_space_unmap(void* ptr, uint64_t size) {
514    void* pagePtr = (void*)(((uintptr_t)ptr) & ~(kPageSize - 1));
515    ::munmap(pagePtr, size);
516}
517
518bool goldfish_address_space_set_subdevice_type(
519    address_space_handle_t handle, GoldfishAddressSpaceSubdeviceType type,
520    address_space_handle_t* handle_out) {
521    struct address_space_ping request;
522    request.metadata = (uint64_t)type;
523    *handle_out = handle;
524    return goldfish_address_space_ping(handle, &request);
525}
526
527bool goldfish_address_space_ping(
528    address_space_handle_t handle,
529    struct address_space_ping* ping) {
530    long res = ioctl_ping(handle, ping);
531
532    if (res) {
533        ALOGE("%s: ping failed: errno: %d\n", __func__, errno);
534        return false;
535    }
536
537    return true;
538}
539