1 /*
2 * Copyright (C) 2016-2020 ARM Limited. All rights reserved.
3 *
4 * Copyright (C) 2008 The Android Open Source Project
5 *
6 * Licensed under the Apache License, Version 2.0 (the "License");
7 * you may not use this file except in compliance with the License.
8 * You may obtain a copy of the License at
9 *
10 * http://www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 */
18
19 #define ATRACE_TAG ATRACE_TAG_GRAPHICS
20
21 #include <string.h>
22 #include <errno.h>
23 #include <inttypes.h>
24 #include <pthread.h>
25 #include <stdlib.h>
26 #include <limits.h>
27
28 #include <cutils/atomic.h>
29 #include <utils/Trace.h>
30
31 #include <linux/dma-buf.h>
32 #include <vector>
33 #include <sys/ioctl.h>
34
35 #include <hardware/hardware.h>
36 #include <hardware/gralloc1.h>
37
38 #include <BufferAllocator/BufferAllocator.h>
39 #include "mali_gralloc_buffer.h"
40 #include "gralloc_helper.h"
41 #include "mali_gralloc_formats.h"
42 #include "mali_gralloc_usages.h"
43 #include "core/format_info.h"
44 #include "core/mali_gralloc_bufferdescriptor.h"
45 #include "core/mali_gralloc_bufferallocation.h"
46
47 #include "mali_gralloc_ion.h"
48
49 #include <array>
50 #include <cassert>
51 #include <string>
52
53 static const char kDmabufSensorDirectHeapName[] = "sensor_direct_heap";
54 static const char kDmabufFaceauthTpuHeapName[] = "faceauth_tpu-secure";
55 static const char kDmabufFaceauthImgHeapName[] = "faimg-secure";
56 static const char kDmabufFaceauthRawImgHeapName[] = "farawimg-secure";
57 static const char kDmabufFaceauthEvalHeapName[] = "faeval-secure";
58 static const char kDmabufFaceauthPrevHeapName[] = "faprev-secure";
59 static const char kDmabufFaceauthModelHeapName[] = "famodel-secure";
60 static const char kDmabufVframeSecureHeapName[] = "vframe-secure";
61 static const char kDmabufVstreamSecureHeapName[] = "vstream-secure";
62 static const char kDmabufVscalerSecureHeapName[] = "vscaler-secure";
63 static const char kDmabufFramebufferSecureHeapName[] = "framebuffer-secure";
64 static const char kDmabufGcmaCameraHeapName[] = "gcma_camera";
65 static const char kDmabufGcmaCameraUncachedHeapName[] = "gcma_camera-uncached";
66
get_allocator()67 BufferAllocator& get_allocator() {
68 static BufferAllocator allocator;
69 return allocator;
70 }
71
find_first_available_heap(const std::initializer_list<std::string> && options)72 std::string find_first_available_heap(const std::initializer_list<std::string>&& options) {
73 static auto available_heaps = BufferAllocator::GetDmabufHeapList();
74
75 for (const auto& heap: options)
76 if (available_heaps.find(heap) != available_heaps.end())
77 return heap;
78
79 return "";
80 }
81
select_dmabuf_heap(uint64_t usage)82 std::string select_dmabuf_heap(uint64_t usage)
83 {
84 struct HeapSpecifier
85 {
86 uint64_t usage_bits;
87 std::string name;
88 };
89
90 static const std::array<HeapSpecifier, 8> exact_usage_heaps =
91 {{
92 // Faceauth heaps
93 { // faceauth_evaluation_heap - used mostly on debug builds
94 GRALLOC_USAGE_PROTECTED | GRALLOC_USAGE_HW_CAMERA_WRITE | GRALLOC_USAGE_HW_CAMERA_READ |
95 GS101_GRALLOC_USAGE_FACEAUTH_RAW_EVAL,
96 kDmabufFaceauthEvalHeapName
97 },
98 { // isp_image_heap
99 GRALLOC_USAGE_PROTECTED | GRALLOC_USAGE_HW_CAMERA_WRITE | GS101_GRALLOC_USAGE_TPU_INPUT,
100 kDmabufFaceauthImgHeapName
101 },
102 { // isp_internal_heap
103 GRALLOC_USAGE_PROTECTED | GRALLOC_USAGE_HW_CAMERA_WRITE | GRALLOC_USAGE_HW_CAMERA_READ,
104 kDmabufFaceauthRawImgHeapName
105 },
106 { // isp_preview_heap
107 GRALLOC_USAGE_PROTECTED | GRALLOC_USAGE_HW_CAMERA_WRITE | GRALLOC_USAGE_HW_COMPOSER |
108 GRALLOC_USAGE_HW_TEXTURE,
109 kDmabufFaceauthPrevHeapName
110 },
111 { // ml_model_heap
112 GRALLOC_USAGE_PROTECTED | GS101_GRALLOC_USAGE_TPU_INPUT,
113 kDmabufFaceauthModelHeapName
114 },
115 { // tpu_heap
116 GRALLOC_USAGE_PROTECTED | GS101_GRALLOC_USAGE_TPU_OUTPUT | GS101_GRALLOC_USAGE_TPU_INPUT,
117 kDmabufFaceauthTpuHeapName
118 },
119
120 {
121 GRALLOC_USAGE_PROTECTED | GRALLOC_USAGE_HW_TEXTURE | GRALLOC_USAGE_HW_RENDER |
122 GRALLOC_USAGE_HW_COMPOSER | GRALLOC_USAGE_HW_FB,
123 find_first_available_heap({kDmabufFramebufferSecureHeapName, kDmabufVframeSecureHeapName})
124 },
125
126 {
127 GRALLOC_USAGE_PROTECTED | GRALLOC_USAGE_HW_TEXTURE | GRALLOC_USAGE_HW_RENDER |
128 GRALLOC_USAGE_HW_COMPOSER,
129 find_first_available_heap({kDmabufFramebufferSecureHeapName, kDmabufVframeSecureHeapName})
130 },
131 }};
132
133 static const std::array<HeapSpecifier, 8> inexact_usage_heaps =
134 {{
135 // If GPU, use vframe-secure
136 {
137 GRALLOC_USAGE_PROTECTED | GRALLOC_USAGE_HW_TEXTURE,
138 kDmabufVframeSecureHeapName
139 },
140 {
141 GRALLOC_USAGE_PROTECTED | GRALLOC_USAGE_HW_RENDER,
142 kDmabufVframeSecureHeapName
143 },
144
145 // If HWC but not GPU
146 {
147 GRALLOC_USAGE_PROTECTED | GRALLOC_USAGE_HW_COMPOSER,
148 kDmabufVscalerSecureHeapName
149 },
150
151 // Catchall for protected
152 {
153 GRALLOC_USAGE_PROTECTED,
154 kDmabufVframeSecureHeapName
155 },
156
157 // Sensor heap
158 {
159 GRALLOC_USAGE_SENSOR_DIRECT_DATA,
160 kDmabufSensorDirectHeapName
161 },
162
163 // Camera GCMA heap
164 {
165 GRALLOC_USAGE_HW_CAMERA_WRITE,
166 find_first_available_heap({kDmabufGcmaCameraUncachedHeapName, kDmabufSystemUncachedHeapName})
167 },
168
169 // Camera GCMA heap
170 {
171 GRALLOC_USAGE_HW_CAMERA_READ,
172 find_first_available_heap({kDmabufGcmaCameraUncachedHeapName, kDmabufSystemUncachedHeapName})
173 },
174
175 // Catchall to system
176 {
177 0,
178 kDmabufSystemUncachedHeapName
179 }
180 }};
181
182 for (const HeapSpecifier &heap : exact_usage_heaps)
183 {
184 if (usage == heap.usage_bits)
185 {
186 return heap.name;
187 }
188 }
189
190 for (const HeapSpecifier &heap : inexact_usage_heaps)
191 {
192 if ((usage & heap.usage_bits) == heap.usage_bits)
193 {
194 if (heap.name == kDmabufGcmaCameraUncachedHeapName &&
195 ((usage & GRALLOC_USAGE_SW_READ_MASK) == GRALLOC_USAGE_SW_READ_OFTEN))
196 return kDmabufGcmaCameraHeapName;
197 else if (heap.name == kDmabufSystemUncachedHeapName &&
198 ((usage & GRALLOC_USAGE_SW_READ_MASK) == GRALLOC_USAGE_SW_READ_OFTEN))
199 return kDmabufSystemHeapName;
200
201 return heap.name;
202 }
203 }
204
205 return "";
206 }
207
alloc_from_dmabuf_heap(uint64_t usage,size_t size,const std::string & buffer_name="",bool use_placeholder=false)208 int alloc_from_dmabuf_heap(uint64_t usage, size_t size, const std::string& buffer_name = "", bool use_placeholder = false)
209 {
210 ATRACE_CALL();
211 if (size == 0) { return -1; }
212
213 auto heap_name = use_placeholder ? "system" : select_dmabuf_heap(usage);
214 if (use_placeholder) size = 1;
215
216 if (heap_name.empty()) {
217 MALI_GRALLOC_LOGW("No heap found for usage: %s (0x%" PRIx64 ")", describe_usage(usage).c_str(), usage);
218 return -EINVAL;
219 }
220
221 std::stringstream tag;
222 tag << "heap: " << heap_name << ", bytes: " << size;
223 ATRACE_NAME(tag.str().c_str());
224 int shared_fd = get_allocator().Alloc(heap_name, size, 0);
225 if (shared_fd < 0)
226 {
227 ALOGE("Allocation failed for heap %s error: %d\n", heap_name.c_str(), shared_fd);
228 }
229
230 if (!buffer_name.empty()) {
231 if (get_allocator().DmabufSetName(shared_fd, buffer_name)) {
232 ALOGW("Unable to set buffer name %s: %s", buffer_name.c_str(), strerror(errno));
233 }
234 }
235
236 return shared_fd;
237 }
238
sync_type_for_flags(const bool read,const bool write)239 SyncType sync_type_for_flags(const bool read, const bool write)
240 {
241 if (read && !write)
242 {
243 return SyncType::kSyncRead;
244 }
245 else if (write && !read)
246 {
247 return SyncType::kSyncWrite;
248 }
249 else
250 {
251 // Deliberately also allowing "not sure" to map to ReadWrite.
252 return SyncType::kSyncReadWrite;
253 }
254 }
255
sync(const int fd,const bool read,const bool write,const bool start)256 int sync(const int fd, const bool read, const bool write, const bool start)
257 {
258 if (start)
259 {
260 return get_allocator().CpuSyncStart(fd, sync_type_for_flags(read, write));
261 }
262 else
263 {
264 return get_allocator().CpuSyncEnd(fd, sync_type_for_flags(read, write));
265 }
266 }
267
mali_gralloc_ion_sync(const private_handle_t * const hnd,const bool read,const bool write,const bool start)268 int mali_gralloc_ion_sync(const private_handle_t * const hnd,
269 const bool read,
270 const bool write,
271 const bool start)
272 {
273 if (hnd == NULL)
274 {
275 return -EINVAL;
276 }
277
278 for (int i = 0; i < hnd->fd_count; i++)
279 {
280 const int fd = hnd->fds[i];
281 if (const int ret = sync(fd, read, write, start))
282 {
283 return ret;
284 }
285 }
286
287 return 0;
288 }
289
290
mali_gralloc_ion_sync_start(const private_handle_t * const hnd,const bool read,const bool write)291 int mali_gralloc_ion_sync_start(const private_handle_t * const hnd,
292 const bool read,
293 const bool write)
294 {
295 return mali_gralloc_ion_sync(hnd, read, write, true);
296 }
297
298
mali_gralloc_ion_sync_end(const private_handle_t * const hnd,const bool read,const bool write)299 int mali_gralloc_ion_sync_end(const private_handle_t * const hnd,
300 const bool read,
301 const bool write)
302 {
303 return mali_gralloc_ion_sync(hnd, read, write, false);
304 }
305
306
mali_gralloc_ion_free(private_handle_t * const hnd)307 void mali_gralloc_ion_free(private_handle_t * const hnd)
308 {
309 for (int i = 0; i < hnd->fd_count; i++)
310 {
311 close(hnd->fds[i]);
312 hnd->fds[i] = -1;
313 }
314 delete hnd;
315 }
316
mali_gralloc_ion_free_internal(buffer_handle_t * const pHandle,const uint32_t num_hnds)317 void mali_gralloc_ion_free_internal(buffer_handle_t * const pHandle,
318 const uint32_t num_hnds)
319 {
320 for (uint32_t i = 0; i < num_hnds; i++)
321 {
322 if (pHandle[i] != NULL)
323 {
324 private_handle_t * const hnd = (private_handle_t * const)pHandle[i];
325 mali_gralloc_ion_free(hnd);
326 }
327 }
328 }
329
mali_gralloc_ion_allocate_attr(private_handle_t * hnd)330 int mali_gralloc_ion_allocate_attr(private_handle_t *hnd)
331 {
332 ATRACE_CALL();
333
334 int idx = hnd->get_share_attr_fd_index();
335 uint64_t usage = GRALLOC_USAGE_SW_READ_OFTEN | GRALLOC_USAGE_SW_WRITE_OFTEN;
336
337 hnd->fds[idx] = alloc_from_dmabuf_heap(usage, hnd->attr_size);
338 if (hnd->fds[idx] < 0)
339 {
340 MALI_GRALLOC_LOGE("ion_alloc failed");
341 return -1;
342 }
343
344 hnd->incr_numfds(1);
345
346 return 0;
347 }
348
349 /*
350 * Allocates ION buffers
351 *
352 * @param descriptors [in] Buffer request descriptors
353 * @param numDescriptors [in] Number of descriptors
354 * @param pHandle [out] Handle for each allocated buffer
355 * @param shared_backend [out] Shared buffers flag
356 *
357 * @return File handle which can be used for allocation, on success
358 * -1, otherwise.
359 */
mali_gralloc_ion_allocate(const gralloc_buffer_descriptor_t * descriptors,uint32_t numDescriptors,buffer_handle_t * pHandle,bool * shared_backend,bool use_placeholder)360 int mali_gralloc_ion_allocate(const gralloc_buffer_descriptor_t *descriptors,
361 uint32_t numDescriptors, buffer_handle_t *pHandle,
362 bool *shared_backend, bool use_placeholder)
363 {
364 ATRACE_CALL();
365 GRALLOC_UNUSED(shared_backend);
366
367 unsigned int priv_heap_flag = 0;
368 uint64_t usage;
369 uint32_t i;
370
371 for (i = 0; i < numDescriptors; i++)
372 {
373 buffer_descriptor_t *bufDescriptor = reinterpret_cast<buffer_descriptor_t *>(descriptors[i]);
374 assert(bufDescriptor);
375 assert(bufDescriptor->fd_count >= 0);
376 assert(bufDescriptor->fd_count <= MAX_FDS);
377
378 auto hnd = new private_handle_t(
379 priv_heap_flag,
380 bufDescriptor->alloc_sizes,
381 bufDescriptor->consumer_usage, bufDescriptor->producer_usage,
382 nullptr, bufDescriptor->fd_count,
383 bufDescriptor->hal_format, bufDescriptor->alloc_format,
384 bufDescriptor->width, bufDescriptor->height, bufDescriptor->pixel_stride,
385 bufDescriptor->layer_count, bufDescriptor->plane_info);
386
387 /* Reset the number of valid filedescriptors, we will increment
388 * it each time a valid fd is added, so we can rely on the
389 * cleanup functions to close open fds. */
390 hnd->set_numfds(0);
391
392 if (nullptr == hnd)
393 {
394 MALI_GRALLOC_LOGE("Private handle could not be created for descriptor:%d in non-shared usecase", i);
395 mali_gralloc_ion_free_internal(pHandle, i);
396 return -1;
397 }
398
399 pHandle[i] = hnd;
400 usage = bufDescriptor->consumer_usage | bufDescriptor->producer_usage;
401
402 for (uint32_t fidx = 0; fidx < bufDescriptor->fd_count; fidx++)
403 {
404 int& fd = hnd->fds[fidx];
405
406 fd = alloc_from_dmabuf_heap(usage, bufDescriptor->alloc_sizes[fidx], bufDescriptor->name, use_placeholder);
407
408 if (fd < 0)
409 {
410 MALI_GRALLOC_LOGE("ion_alloc failed for fds[%u] = %d", fidx, fd);
411 mali_gralloc_ion_free_internal(pHandle, i + 1);
412 return -1;
413 }
414
415 hnd->incr_numfds(1);
416 }
417 }
418
419 if (use_placeholder) return 0;
420
421 #if defined(GRALLOC_INIT_AFBC) && (GRALLOC_INIT_AFBC == 1)
422 ATRACE_NAME("AFBC init block");
423 unsigned char *cpu_ptr = NULL;
424 for (i = 0; i < numDescriptors; i++)
425 {
426 buffer_descriptor_t *bufDescriptor = reinterpret_cast<buffer_descriptor_t *>(descriptors[i]);
427 const private_handle_t *hnd = static_cast<const private_handle_t *>(pHandle[i]);
428
429 usage = bufDescriptor->consumer_usage | bufDescriptor->producer_usage;
430
431 if ((bufDescriptor->alloc_format & MALI_GRALLOC_INTFMT_AFBCENABLE_MASK)
432 && !(usage & GRALLOC_USAGE_PROTECTED))
433 {
434 {
435 ATRACE_NAME("mmap");
436 /* TODO: only map for AFBC buffers */
437 cpu_ptr =
438 (unsigned char *)mmap(NULL, bufDescriptor->alloc_sizes[0], PROT_READ | PROT_WRITE, MAP_SHARED, hnd->fds[0], 0);
439
440 if (MAP_FAILED == cpu_ptr)
441 {
442 MALI_GRALLOC_LOGE("mmap failed for fd ( %d )", hnd->fds[0]);
443 mali_gralloc_ion_free_internal(pHandle, numDescriptors);
444 return -1;
445 }
446
447 mali_gralloc_ion_sync_start(hnd, true, true);
448 }
449
450 {
451 ATRACE_NAME("data init");
452 /* For separated plane YUV, there is a header to initialise per plane. */
453 const plane_info_t *plane_info = bufDescriptor->plane_info;
454 assert(plane_info);
455 const bool is_multi_plane = hnd->is_multi_plane();
456 /* We'll need bpp for initialize header pointers*/
457 const uint32_t base_format = bufDescriptor->alloc_format & MALI_GRALLOC_INTFMT_FMT_MASK;
458 int32_t format_idx = get_format_index(base_format);
459 if (format_idx == -1)
460 {
461 MALI_GRALLOC_LOGE("Unknown format %" PRIx32 ", AFBC initialization is impossible", base_format);
462 mali_gralloc_ion_free_internal(pHandle, numDescriptors);
463 return -1;
464 }
465 for (int i = 0; i < MAX_PLANES && (i == 0 || plane_info[i].byte_stride != 0); i++)
466 {
467 init_afbc(cpu_ptr + plane_info[i].offset,
468 bufDescriptor->alloc_format,
469 is_multi_plane,
470 formats[format_idx].bpp_afbc[i],
471 plane_info[i].alloc_width,
472 plane_info[i].alloc_height);
473 }
474 }
475
476 {
477 ATRACE_NAME("munmap");
478 mali_gralloc_ion_sync_end(hnd, true, true);
479 munmap(cpu_ptr, bufDescriptor->alloc_sizes[0]);
480 }
481 }
482 }
483 #endif
484
485 return 0;
486 }
487
mali_gralloc_ion_map(private_handle_t * hnd)488 std::array<void*, MAX_BUFFER_FDS> mali_gralloc_ion_map(private_handle_t *hnd)
489 {
490 std::array<void*, MAX_BUFFER_FDS> vaddrs;
491 vaddrs.fill(nullptr);
492
493 uint64_t usage = hnd->producer_usage | hnd->consumer_usage;
494 /* Do not allow cpu access to secure buffers */
495 if (usage & (GRALLOC_USAGE_PROTECTED | GRALLOC_USAGE_NOZEROED)
496 && !(usage & GRALLOC_USAGE_PRIVATE_NONSECURE))
497 {
498 return vaddrs;
499 }
500
501 for (int fidx = 0; fidx < hnd->fd_count; fidx++) {
502 unsigned char *mappedAddress =
503 (unsigned char *)mmap(NULL, hnd->alloc_sizes[fidx], PROT_READ | PROT_WRITE,
504 MAP_SHARED, hnd->fds[fidx], 0);
505
506 if (MAP_FAILED == mappedAddress)
507 {
508 int err = errno;
509 MALI_GRALLOC_LOGE("mmap( fds[%d]:%d size:%" PRIu64 " ) failed with %s",
510 fidx, hnd->fds[fidx], hnd->alloc_sizes[fidx], strerror(err));
511 hnd->dump("map fail");
512
513 for (int cidx = 0; cidx < fidx; fidx++)
514 {
515 munmap((void*)vaddrs[cidx], hnd->alloc_sizes[cidx]);
516 vaddrs[cidx] = 0;
517 }
518
519 return vaddrs;
520 }
521
522 vaddrs[fidx] = mappedAddress;
523 }
524
525 return vaddrs;
526 }
527
mali_gralloc_ion_unmap(private_handle_t * hnd,std::array<void *,MAX_BUFFER_FDS> & vaddrs)528 void mali_gralloc_ion_unmap(private_handle_t *hnd, std::array<void*, MAX_BUFFER_FDS>& vaddrs)
529 {
530 for (int i = 0; i < hnd->fd_count; i++)
531 {
532 int err = 0;
533
534 if (vaddrs[i])
535 {
536 err = munmap(vaddrs[i], hnd->alloc_sizes[i]);
537 }
538
539 if (err)
540 {
541 MALI_GRALLOC_LOGE("Could not munmap base:%p size:%" PRIu64 " '%s'",
542 (void*)vaddrs[i], hnd->alloc_sizes[i], strerror(errno));
543 }
544 else
545 {
546 vaddrs[i] = 0;
547 }
548 }
549
550 hnd->cpu_read = 0;
551 hnd->cpu_write = 0;
552 }
553