1 /*
2 * Copyright © 2022 Collabora Ltd
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include "vk_meta_private.h"
25
26 #include "vk_command_buffer.h"
27 #include "vk_device.h"
28 #include "vk_format.h"
29 #include "vk_pipeline.h"
30
31 #include "nir_builder.h"
32
33 struct vk_meta_blit_key {
34 enum vk_meta_object_key_type key_type;
35 enum glsl_sampler_dim dim;
36 VkSampleCountFlagBits src_samples;
37 VkResolveModeFlagBits resolve_mode;
38 VkResolveModeFlagBits stencil_resolve_mode;
39 bool stencil_as_discard;
40 VkFormat dst_format;
41 VkImageAspectFlags aspects;
42 };
43
44 static enum glsl_sampler_dim
vk_image_sampler_dim(const struct vk_image * image)45 vk_image_sampler_dim(const struct vk_image *image)
46 {
47 switch (image->image_type) {
48 case VK_IMAGE_TYPE_1D: return GLSL_SAMPLER_DIM_1D;
49 case VK_IMAGE_TYPE_2D:
50 if (image->samples > 1)
51 return GLSL_SAMPLER_DIM_MS;
52 else
53 return GLSL_SAMPLER_DIM_2D;
54 case VK_IMAGE_TYPE_3D: return GLSL_SAMPLER_DIM_3D;
55 default: unreachable("Invalid image type");
56 }
57 }
58
59 enum blit_desc_binding {
60 BLIT_DESC_BINDING_SAMPLER,
61 BLIT_DESC_BINDING_COLOR,
62 BLIT_DESC_BINDING_DEPTH,
63 BLIT_DESC_BINDING_STENCIL,
64 };
65
66 static enum blit_desc_binding
aspect_to_tex_binding(VkImageAspectFlagBits aspect)67 aspect_to_tex_binding(VkImageAspectFlagBits aspect)
68 {
69 switch (aspect) {
70 case VK_IMAGE_ASPECT_COLOR_BIT: return BLIT_DESC_BINDING_COLOR;
71 case VK_IMAGE_ASPECT_DEPTH_BIT: return BLIT_DESC_BINDING_DEPTH;
72 case VK_IMAGE_ASPECT_STENCIL_BIT: return BLIT_DESC_BINDING_STENCIL;
73 default: unreachable("Unsupported aspect");
74 }
75 }
76
77 struct vk_meta_blit_push_data {
78 float x_off, y_off, x_scale, y_scale;
79 float z_off, z_scale;
80 int32_t arr_delta;
81 uint32_t stencil_bit;
82 };
83
84 static inline void
compute_off_scale(uint32_t src_level_size,uint32_t src0,uint32_t src1,uint32_t dst0,uint32_t dst1,uint32_t * dst0_out,uint32_t * dst1_out,float * off_out,float * scale_out)85 compute_off_scale(uint32_t src_level_size,
86 uint32_t src0, uint32_t src1,
87 uint32_t dst0, uint32_t dst1,
88 uint32_t *dst0_out, uint32_t *dst1_out,
89 float *off_out, float *scale_out)
90 {
91 assert(src0 <= src_level_size && src1 <= src_level_size);
92
93 if (dst0 < dst1) {
94 *dst0_out = dst0;
95 *dst1_out = dst1;
96 } else {
97 *dst0_out = dst1;
98 *dst1_out = dst0;
99
100 /* Flip the source region */
101 uint32_t tmp = src0;
102 src0 = src1;
103 src1 = tmp;
104 }
105
106 double src_region_size = (double)src1 - (double)src0;
107 assert(src_region_size != 0);
108
109 double dst_region_size = (double)*dst1_out - (double)*dst0_out;
110 assert(dst_region_size > 0);
111
112 double src_offset = src0 / (double)src_level_size;
113 double dst_scale = src_region_size / (src_level_size * dst_region_size);
114 double dst_offset = (double)*dst0_out * dst_scale;
115
116 *off_out = src_offset - dst_offset;
117 *scale_out = dst_scale;
118 }
119
120 static inline nir_def *
load_struct_var(nir_builder * b,nir_variable * var,uint32_t field)121 load_struct_var(nir_builder *b, nir_variable *var, uint32_t field)
122 {
123 nir_deref_instr *deref =
124 nir_build_deref_struct(b, nir_build_deref_var(b, var), field);
125 return nir_load_deref(b, deref);
126 }
127
128 static nir_def *
build_tex_resolve(nir_builder * b,nir_deref_instr * t,nir_def * coord,VkSampleCountFlagBits samples,VkResolveModeFlagBits resolve_mode)129 build_tex_resolve(nir_builder *b, nir_deref_instr *t,
130 nir_def *coord,
131 VkSampleCountFlagBits samples,
132 VkResolveModeFlagBits resolve_mode)
133 {
134 nir_def *accum = nir_txf_ms_deref(b, t, coord, nir_imm_int(b, 0));
135 if (resolve_mode == VK_RESOLVE_MODE_SAMPLE_ZERO_BIT)
136 return accum;
137
138 const enum glsl_base_type base_type =
139 glsl_get_sampler_result_type(t->type);
140
141 for (unsigned i = 1; i < samples; i++) {
142 nir_def *val = nir_txf_ms_deref(b, t, coord, nir_imm_int(b, i));
143 switch (resolve_mode) {
144 case VK_RESOLVE_MODE_AVERAGE_BIT:
145 assert(base_type == GLSL_TYPE_FLOAT);
146 accum = nir_fadd(b, accum, val);
147 break;
148
149 case VK_RESOLVE_MODE_MIN_BIT:
150 switch (base_type) {
151 case GLSL_TYPE_UINT:
152 accum = nir_umin(b, accum, val);
153 break;
154 case GLSL_TYPE_INT:
155 accum = nir_imin(b, accum, val);
156 break;
157 case GLSL_TYPE_FLOAT:
158 accum = nir_fmin(b, accum, val);
159 break;
160 default:
161 unreachable("Invalid sample result type");
162 }
163 break;
164
165 case VK_RESOLVE_MODE_MAX_BIT:
166 switch (base_type) {
167 case GLSL_TYPE_UINT:
168 accum = nir_umax(b, accum, val);
169 break;
170 case GLSL_TYPE_INT:
171 accum = nir_imax(b, accum, val);
172 break;
173 case GLSL_TYPE_FLOAT:
174 accum = nir_fmax(b, accum, val);
175 break;
176 default:
177 unreachable("Invalid sample result type");
178 }
179 break;
180
181 default:
182 unreachable("Unsupported resolve mode");
183 }
184 }
185
186 if (resolve_mode == VK_RESOLVE_MODE_AVERAGE_BIT)
187 accum = nir_fmul_imm(b, accum, 1.0 / samples);
188
189 return accum;
190 }
191
192 static nir_shader *
build_blit_shader(const struct vk_meta_blit_key * key)193 build_blit_shader(const struct vk_meta_blit_key *key)
194 {
195 nir_builder build;
196 if (key->resolve_mode || key->stencil_resolve_mode) {
197 build = nir_builder_init_simple_shader(MESA_SHADER_FRAGMENT, NULL,
198 "vk-meta-resolve");
199 } else {
200 build = nir_builder_init_simple_shader(MESA_SHADER_FRAGMENT,
201 NULL, "vk-meta-blit");
202 }
203 nir_builder *b = &build;
204
205 struct glsl_struct_field push_fields[] = {
206 { .type = glsl_vec4_type(), .name = "xy_xform", .offset = 0 },
207 { .type = glsl_vec4_type(), .name = "z_xform", .offset = 16 },
208 };
209 const struct glsl_type *push_iface_type =
210 glsl_interface_type(push_fields, ARRAY_SIZE(push_fields),
211 GLSL_INTERFACE_PACKING_STD140,
212 false /* row_major */, "push");
213 nir_variable *push = nir_variable_create(b->shader, nir_var_mem_push_const,
214 push_iface_type, "push");
215
216 nir_def *xy_xform = load_struct_var(b, push, 0);
217 nir_def *xy_off = nir_channels(b, xy_xform, 3 << 0);
218 nir_def *xy_scale = nir_channels(b, xy_xform, 3 << 2);
219
220 nir_def *out_coord_xy = nir_load_frag_coord(b);
221 out_coord_xy = nir_trim_vector(b, out_coord_xy, 2);
222 nir_def *src_coord_xy = nir_ffma(b, out_coord_xy, xy_scale, xy_off);
223
224 nir_def *z_xform = load_struct_var(b, push, 1);
225 nir_def *out_layer = nir_load_layer_id(b);
226 nir_def *src_coord;
227 if (key->dim == GLSL_SAMPLER_DIM_3D) {
228 nir_def *z_off = nir_channel(b, z_xform, 0);
229 nir_def *z_scale = nir_channel(b, z_xform, 1);
230 nir_def *out_coord_z = nir_fadd_imm(b, nir_u2f32(b, out_layer), 0.5);
231 nir_def *src_coord_z = nir_ffma(b, out_coord_z, z_scale, z_off);
232 src_coord = nir_vec3(b, nir_channel(b, src_coord_xy, 0),
233 nir_channel(b, src_coord_xy, 1),
234 src_coord_z);
235 } else {
236 nir_def *arr_delta = nir_channel(b, z_xform, 2);
237 nir_def *in_layer = nir_iadd(b, out_layer, arr_delta);
238 if (key->dim == GLSL_SAMPLER_DIM_1D) {
239 src_coord = nir_vec2(b, nir_channel(b, src_coord_xy, 0),
240 nir_u2f32(b, in_layer));
241 } else {
242 assert(key->dim == GLSL_SAMPLER_DIM_2D ||
243 key->dim == GLSL_SAMPLER_DIM_MS);
244 src_coord = nir_vec3(b, nir_channel(b, src_coord_xy, 0),
245 nir_channel(b, src_coord_xy, 1),
246 nir_u2f32(b, in_layer));
247 }
248 }
249
250 nir_variable *sampler = nir_variable_create(b->shader, nir_var_uniform,
251 glsl_bare_sampler_type(), NULL);
252 sampler->data.descriptor_set = 0;
253 sampler->data.binding = BLIT_DESC_BINDING_SAMPLER;
254 nir_deref_instr *s = nir_build_deref_var(b, sampler);
255
256 u_foreach_bit(a, key->aspects) {
257 VkImageAspectFlagBits aspect = (1 << a);
258
259 enum glsl_base_type base_type;
260 unsigned out_location, out_comps;
261 const char *tex_name, *out_name;
262 VkResolveModeFlagBits resolve_mode;
263 switch (aspect) {
264 case VK_IMAGE_ASPECT_COLOR_BIT:
265 tex_name = "color_tex";
266 if (vk_format_is_int(key->dst_format))
267 base_type = GLSL_TYPE_INT;
268 else if (vk_format_is_uint(key->dst_format))
269 base_type = GLSL_TYPE_UINT;
270 else
271 base_type = GLSL_TYPE_FLOAT;
272 resolve_mode = key->resolve_mode;
273 out_name = "gl_FragData[0]";
274 out_location = FRAG_RESULT_DATA0;
275 out_comps = 4;
276 break;
277 case VK_IMAGE_ASPECT_DEPTH_BIT:
278 tex_name = "depth_tex";
279 base_type = GLSL_TYPE_FLOAT;
280 resolve_mode = key->resolve_mode;
281 out_name = "gl_FragDepth";
282 out_location = FRAG_RESULT_DEPTH;
283 out_comps = 1;
284 break;
285 case VK_IMAGE_ASPECT_STENCIL_BIT:
286 tex_name = "stencil_tex";
287 base_type = GLSL_TYPE_UINT;
288 resolve_mode = key->stencil_resolve_mode;
289 out_name = "gl_FragStencilRef";
290 out_location = FRAG_RESULT_STENCIL;
291 out_comps = 1;
292 break;
293 default:
294 unreachable("Unsupported aspect");
295 }
296
297 const bool is_array = key->dim != GLSL_SAMPLER_DIM_3D;
298 const struct glsl_type *texture_type =
299 glsl_sampler_type(key->dim, false, is_array, base_type);
300 nir_variable *texture = nir_variable_create(b->shader, nir_var_uniform,
301 texture_type, tex_name);
302 texture->data.descriptor_set = 0;
303 texture->data.binding = aspect_to_tex_binding(aspect);
304 nir_deref_instr *t = nir_build_deref_var(b, texture);
305
306 nir_def *val;
307 if (resolve_mode == VK_RESOLVE_MODE_NONE) {
308 val = nir_txl_deref(b, t, s, src_coord, nir_imm_float(b, 0));
309 } else {
310 val = build_tex_resolve(b, t, nir_f2u32(b, src_coord),
311 key->src_samples, resolve_mode);
312 }
313 val = nir_trim_vector(b, val, out_comps);
314
315 if (key->stencil_as_discard) {
316 assert(key->aspects == VK_IMAGE_ASPECT_STENCIL_BIT);
317 nir_def *stencil_bit = nir_channel(b, z_xform, 3);
318 nir_discard_if(b, nir_ieq(b, nir_iand(b, val, stencil_bit),
319 nir_imm_int(b, 0)));
320 } else {
321 const struct glsl_type *out_type =
322 glsl_vector_type(base_type, out_comps);
323 nir_variable *out = nir_variable_create(b->shader, nir_var_shader_out,
324 out_type, out_name);
325 out->data.location = out_location;
326
327 nir_store_var(b, out, val, BITFIELD_MASK(out_comps));
328 }
329 }
330
331 return b->shader;
332 }
333
334 static VkResult
get_blit_pipeline_layout(struct vk_device * device,struct vk_meta_device * meta,VkPipelineLayout * layout_out)335 get_blit_pipeline_layout(struct vk_device *device,
336 struct vk_meta_device *meta,
337 VkPipelineLayout *layout_out)
338 {
339 const char key[] = "vk-meta-blit-pipeline-layout";
340
341 const VkDescriptorSetLayoutBinding bindings[] = {{
342 .binding = BLIT_DESC_BINDING_SAMPLER,
343 .descriptorType = VK_DESCRIPTOR_TYPE_SAMPLER,
344 .descriptorCount = 1,
345 .stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT,
346 }, {
347 .binding = BLIT_DESC_BINDING_COLOR,
348 .descriptorType = VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE,
349 .descriptorCount = 1,
350 .stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT,
351 }, {
352 .binding = BLIT_DESC_BINDING_DEPTH,
353 .descriptorType = VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE,
354 .descriptorCount = 1,
355 .stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT,
356 }, {
357 .binding = BLIT_DESC_BINDING_STENCIL,
358 .descriptorType = VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE,
359 .descriptorCount = 1,
360 .stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT,
361 }};
362
363 const VkDescriptorSetLayoutCreateInfo desc_info = {
364 .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
365 .bindingCount = ARRAY_SIZE(bindings),
366 .pBindings = bindings,
367 };
368
369 const VkPushConstantRange push_range = {
370 .stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT,
371 .offset = 0,
372 .size = sizeof(struct vk_meta_blit_push_data),
373 };
374
375 return vk_meta_get_pipeline_layout(device, meta, &desc_info, &push_range,
376 key, sizeof(key), layout_out);
377 }
378
379 static VkResult
get_blit_pipeline(struct vk_device * device,struct vk_meta_device * meta,const struct vk_meta_blit_key * key,VkPipelineLayout layout,VkPipeline * pipeline_out)380 get_blit_pipeline(struct vk_device *device,
381 struct vk_meta_device *meta,
382 const struct vk_meta_blit_key *key,
383 VkPipelineLayout layout,
384 VkPipeline *pipeline_out)
385 {
386 VkPipeline from_cache = vk_meta_lookup_pipeline(meta, key, sizeof(*key));
387 if (from_cache != VK_NULL_HANDLE) {
388 *pipeline_out = from_cache;
389 return VK_SUCCESS;
390 }
391
392 const VkPipelineShaderStageNirCreateInfoMESA fs_nir_info = {
393 .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_NIR_CREATE_INFO_MESA,
394 .nir = build_blit_shader(key),
395 };
396 const VkPipelineShaderStageCreateInfo fs_info = {
397 .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
398 .pNext = &fs_nir_info,
399 .stage = VK_SHADER_STAGE_FRAGMENT_BIT,
400 .pName = "main",
401 };
402
403 VkPipelineDepthStencilStateCreateInfo ds_info = {
404 .sType = VK_STRUCTURE_TYPE_PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO,
405 };
406 VkDynamicState dyn_tmp;
407 VkPipelineDynamicStateCreateInfo dyn_info = {
408 .sType = VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO,
409 };
410 struct vk_meta_rendering_info render = {
411 .samples = 1,
412 };
413 if (key->aspects & VK_IMAGE_ASPECT_COLOR_BIT) {
414 render.color_attachment_count = 1;
415 render.color_attachment_formats[0] = key->dst_format;
416 }
417 if (key->aspects & VK_IMAGE_ASPECT_DEPTH_BIT) {
418 ds_info.depthTestEnable = VK_TRUE;
419 ds_info.depthWriteEnable = VK_TRUE;
420 ds_info.depthCompareOp = VK_COMPARE_OP_ALWAYS;
421 render.depth_attachment_format = key->dst_format;
422 }
423 if (key->aspects & VK_IMAGE_ASPECT_STENCIL_BIT) {
424 ds_info.stencilTestEnable = VK_TRUE;
425 ds_info.front.compareOp = VK_COMPARE_OP_ALWAYS;
426 ds_info.front.passOp = VK_STENCIL_OP_REPLACE;
427 ds_info.front.compareMask = ~0u;
428 ds_info.front.writeMask = ~0u;
429 ds_info.front.reference = ~0;
430 ds_info.back = ds_info.front;
431 if (key->stencil_as_discard) {
432 dyn_tmp = VK_DYNAMIC_STATE_STENCIL_WRITE_MASK;
433 dyn_info.dynamicStateCount = 1;
434 dyn_info.pDynamicStates = &dyn_tmp;
435 }
436 render.stencil_attachment_format = key->dst_format;
437 }
438
439 const VkGraphicsPipelineCreateInfo info = {
440 .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO,
441 .stageCount = 1,
442 .pStages = &fs_info,
443 .pDepthStencilState = &ds_info,
444 .pDynamicState = &dyn_info,
445 .layout = layout,
446 };
447
448 VkResult result = vk_meta_create_graphics_pipeline(device, meta, &info,
449 &render,
450 key, sizeof(*key),
451 pipeline_out);
452 ralloc_free(fs_nir_info.nir);
453
454 return result;
455 }
456
457 static VkResult
get_blit_sampler(struct vk_device * device,struct vk_meta_device * meta,VkFilter filter,VkSampler * sampler_out)458 get_blit_sampler(struct vk_device *device,
459 struct vk_meta_device *meta,
460 VkFilter filter,
461 VkSampler *sampler_out)
462 {
463 struct {
464 enum vk_meta_object_key_type key_type;
465 VkFilter filter;
466 } key;
467
468 memset(&key, 0, sizeof(key));
469 key.key_type = VK_META_OBJECT_KEY_BLIT_SAMPLER;
470 key.filter = filter;
471
472 VkSampler from_cache = vk_meta_lookup_sampler(meta, &key, sizeof(key));
473 if (from_cache != VK_NULL_HANDLE) {
474 *sampler_out = from_cache;
475 return VK_SUCCESS;
476 }
477
478 const VkSamplerCreateInfo info = {
479 .sType = VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO,
480 .magFilter = filter,
481 .minFilter = filter,
482 .mipmapMode = VK_SAMPLER_MIPMAP_MODE_NEAREST,
483 .addressModeU = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE,
484 .addressModeV = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE,
485 .addressModeW = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE,
486 .unnormalizedCoordinates = VK_FALSE,
487 };
488
489 return vk_meta_create_sampler(device, meta, &info,
490 &key, sizeof(key), sampler_out);
491 }
492
493 static void
do_blit(struct vk_command_buffer * cmd,struct vk_meta_device * meta,struct vk_image * src_image,VkFormat src_format,VkImageLayout src_image_layout,VkImageSubresourceLayers src_subres,struct vk_image * dst_image,VkFormat dst_format,VkImageLayout dst_image_layout,VkImageSubresourceLayers dst_subres,VkSampler sampler,struct vk_meta_blit_key * key,struct vk_meta_blit_push_data * push,const struct vk_meta_rect * dst_rect,uint32_t dst_layer_count)494 do_blit(struct vk_command_buffer *cmd,
495 struct vk_meta_device *meta,
496 struct vk_image *src_image,
497 VkFormat src_format,
498 VkImageLayout src_image_layout,
499 VkImageSubresourceLayers src_subres,
500 struct vk_image *dst_image,
501 VkFormat dst_format,
502 VkImageLayout dst_image_layout,
503 VkImageSubresourceLayers dst_subres,
504 VkSampler sampler,
505 struct vk_meta_blit_key *key,
506 struct vk_meta_blit_push_data *push,
507 const struct vk_meta_rect *dst_rect,
508 uint32_t dst_layer_count)
509 {
510 struct vk_device *device = cmd->base.device;
511 const struct vk_device_dispatch_table *disp = &device->dispatch_table;
512 VkResult result;
513
514 VkPipelineLayout pipeline_layout;
515 result = get_blit_pipeline_layout(device, meta, &pipeline_layout);
516 if (unlikely(result != VK_SUCCESS)) {
517 vk_command_buffer_set_error(cmd, result);
518 return;
519 }
520
521 uint32_t desc_count = 0;
522 VkDescriptorImageInfo image_infos[3];
523 VkWriteDescriptorSet desc_writes[3];
524
525 if (sampler != VK_NULL_HANDLE) {
526 image_infos[desc_count] = (VkDescriptorImageInfo) {
527 .sampler = sampler,
528 };
529 desc_writes[desc_count] = (VkWriteDescriptorSet) {
530 .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
531 .dstBinding = BLIT_DESC_BINDING_SAMPLER,
532 .descriptorType = VK_DESCRIPTOR_TYPE_SAMPLER,
533 .descriptorCount = 1,
534 .pImageInfo = &image_infos[desc_count],
535 };
536 desc_count++;
537 }
538
539 u_foreach_bit(a, src_subres.aspectMask) {
540 VkImageAspectFlagBits aspect = (1 << a);
541
542 VkImageView src_view;
543 const VkImageViewUsageCreateInfo src_view_usage = {
544 .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_USAGE_CREATE_INFO,
545 .usage = VK_IMAGE_USAGE_SAMPLED_BIT,
546 };
547 const VkImageViewCreateInfo src_view_info = {
548 .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO,
549 .pNext = &src_view_usage,
550 .image = vk_image_to_handle(src_image),
551 .viewType = vk_image_sampled_view_type(src_image),
552 .format = src_format,
553 .subresourceRange = {
554 .aspectMask = aspect,
555 .baseMipLevel = src_subres.mipLevel,
556 .levelCount = 1,
557 .baseArrayLayer = src_subres.baseArrayLayer,
558 .layerCount = src_subres.layerCount,
559 },
560 };
561 result = vk_meta_create_image_view(cmd, meta, &src_view_info,
562 &src_view);
563 if (unlikely(result != VK_SUCCESS)) {
564 vk_command_buffer_set_error(cmd, result);
565 return;
566 }
567
568 assert(desc_count < ARRAY_SIZE(image_infos));
569 assert(desc_count < ARRAY_SIZE(desc_writes));
570 image_infos[desc_count] = (VkDescriptorImageInfo) {
571 .imageView = src_view,
572 };
573 desc_writes[desc_count] = (VkWriteDescriptorSet) {
574 .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
575 .dstBinding = aspect_to_tex_binding(aspect),
576 .descriptorType = VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE,
577 .descriptorCount = 1,
578 .pImageInfo = &image_infos[desc_count],
579 };
580 desc_count++;
581 }
582
583 disp->CmdPushDescriptorSetKHR(vk_command_buffer_to_handle(cmd),
584 VK_PIPELINE_BIND_POINT_GRAPHICS,
585 pipeline_layout, 0,
586 desc_count, desc_writes);
587
588 assert(dst_subres.aspectMask == src_subres.aspectMask);
589 VkImageAspectFlags aspects_left = dst_subres.aspectMask;
590
591 while (aspects_left) {
592 key->aspects = aspects_left;
593
594 /* If we need to write stencil via iterative discard, it has to be
595 * written by itself because otherwise the discards would also throw
596 * away color or depth data.
597 */
598 if ((key->aspects & VK_IMAGE_ASPECT_STENCIL_BIT) &&
599 key->aspects != VK_IMAGE_ASPECT_STENCIL_BIT &&
600 !meta->use_stencil_export)
601 key->aspects &= ~VK_IMAGE_ASPECT_STENCIL_BIT;
602
603 key->stencil_as_discard = key->aspects == VK_IMAGE_ASPECT_STENCIL_BIT &&
604 !meta->use_stencil_export;
605
606 VkImageView dst_view;
607 const VkImageViewUsageCreateInfo dst_view_usage = {
608 .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_USAGE_CREATE_INFO,
609 .usage = (key->aspects & VK_IMAGE_ASPECT_COLOR_BIT) ?
610 VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT :
611 VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT,
612 };
613 const VkImageViewCreateInfo dst_view_info = {
614 .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO,
615 .pNext = &dst_view_usage,
616 .image = vk_image_to_handle(dst_image),
617 .viewType = vk_image_sampled_view_type(dst_image),
618 .format = dst_format,
619 .subresourceRange = {
620 .aspectMask = dst_subres.aspectMask,
621 .baseMipLevel = dst_subres.mipLevel,
622 .levelCount = 1,
623 .baseArrayLayer = dst_subres.baseArrayLayer,
624 .layerCount = dst_subres.layerCount,
625 },
626 };
627 result = vk_meta_create_image_view(cmd, meta, &dst_view_info,
628 &dst_view);
629 if (unlikely(result != VK_SUCCESS)) {
630 vk_command_buffer_set_error(cmd, result);
631 return;
632 }
633
634 const VkRenderingAttachmentInfo vk_att = {
635 .sType = VK_STRUCTURE_TYPE_RENDERING_ATTACHMENT_INFO,
636 .imageView = dst_view,
637 .imageLayout = dst_image_layout,
638 .loadOp = key->stencil_as_discard ? VK_ATTACHMENT_LOAD_OP_CLEAR :
639 VK_ATTACHMENT_LOAD_OP_DONT_CARE,
640 .storeOp = VK_ATTACHMENT_STORE_OP_STORE,
641 };
642 VkRenderingInfo vk_render = {
643 .sType = VK_STRUCTURE_TYPE_RENDERING_INFO,
644 .renderArea = {
645 .offset = {
646 dst_rect->x0,
647 dst_rect->y0
648 },
649 .extent = {
650 dst_rect->x1 - dst_rect->x0,
651 dst_rect->y1 - dst_rect->y0
652 },
653 },
654 .layerCount = dst_rect->layer + dst_layer_count,
655 };
656
657 if (key->aspects & VK_IMAGE_ASPECT_COLOR_BIT) {
658 vk_render.colorAttachmentCount = 1;
659 vk_render.pColorAttachments = &vk_att;
660 }
661 if (key->aspects & VK_IMAGE_ASPECT_DEPTH_BIT)
662 vk_render.pDepthAttachment = &vk_att;
663 if (key->aspects & VK_IMAGE_ASPECT_STENCIL_BIT)
664 vk_render.pStencilAttachment = &vk_att;
665
666 disp->CmdBeginRendering(vk_command_buffer_to_handle(cmd), &vk_render);
667
668 VkPipeline pipeline;
669 result = get_blit_pipeline(device, meta, key,
670 pipeline_layout, &pipeline);
671 if (unlikely(result != VK_SUCCESS)) {
672 vk_command_buffer_set_error(cmd, result);
673 return;
674 }
675
676 disp->CmdBindPipeline(vk_command_buffer_to_handle(cmd),
677 VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline);
678
679 if (key->stencil_as_discard) {
680 for (uint32_t i = 0; i < 8; i++) {
681 push->stencil_bit = BITFIELD_BIT(i);
682 disp->CmdPushConstants(vk_command_buffer_to_handle(cmd),
683 pipeline_layout,
684 VK_SHADER_STAGE_FRAGMENT_BIT,
685 0, sizeof(*push), push);
686
687 disp->CmdSetStencilWriteMask(vk_command_buffer_to_handle(cmd),
688 VK_STENCIL_FACE_FRONT_AND_BACK,
689 push->stencil_bit);
690
691 meta->cmd_draw_volume(cmd, meta, dst_rect, dst_layer_count);
692 }
693 } else {
694 disp->CmdPushConstants(vk_command_buffer_to_handle(cmd),
695 pipeline_layout,
696 VK_SHADER_STAGE_FRAGMENT_BIT,
697 0, sizeof(*push), push);
698
699 meta->cmd_draw_volume(cmd, meta, dst_rect, dst_layer_count);
700 }
701
702 disp->CmdEndRendering(vk_command_buffer_to_handle(cmd));
703
704 aspects_left &= ~key->aspects;
705 }
706 }
707
708 void
vk_meta_blit_image(struct vk_command_buffer * cmd,struct vk_meta_device * meta,struct vk_image * src_image,VkFormat src_format,VkImageLayout src_image_layout,struct vk_image * dst_image,VkFormat dst_format,VkImageLayout dst_image_layout,uint32_t region_count,const VkImageBlit2 * regions,VkFilter filter)709 vk_meta_blit_image(struct vk_command_buffer *cmd,
710 struct vk_meta_device *meta,
711 struct vk_image *src_image,
712 VkFormat src_format,
713 VkImageLayout src_image_layout,
714 struct vk_image *dst_image,
715 VkFormat dst_format,
716 VkImageLayout dst_image_layout,
717 uint32_t region_count,
718 const VkImageBlit2 *regions,
719 VkFilter filter)
720 {
721 struct vk_device *device = cmd->base.device;
722 VkResult result;
723
724 VkSampler sampler;
725 result = get_blit_sampler(device, meta, filter, &sampler);
726 if (unlikely(result != VK_SUCCESS)) {
727 vk_command_buffer_set_error(cmd, result);
728 return;
729 }
730
731 struct vk_meta_blit_key key;
732 memset(&key, 0, sizeof(key));
733 key.key_type = VK_META_OBJECT_KEY_BLIT_PIPELINE;
734 key.src_samples = src_image->samples;
735 key.dim = vk_image_sampler_dim(src_image);
736 key.dst_format = dst_format;
737
738 for (uint32_t r = 0; r < region_count; r++) {
739 struct vk_meta_blit_push_data push = {0};
740 struct vk_meta_rect dst_rect = {0};
741
742 uint32_t src_level = regions[r].srcSubresource.mipLevel;
743 VkExtent3D src_extent = vk_image_mip_level_extent(src_image, src_level);
744
745 compute_off_scale(src_extent.width,
746 regions[r].srcOffsets[0].x,
747 regions[r].srcOffsets[1].x,
748 regions[r].dstOffsets[0].x,
749 regions[r].dstOffsets[1].x,
750 &dst_rect.x0, &dst_rect.x1,
751 &push.x_off, &push.x_scale);
752 compute_off_scale(src_extent.height,
753 regions[r].srcOffsets[0].y,
754 regions[r].srcOffsets[1].y,
755 regions[r].dstOffsets[0].y,
756 regions[r].dstOffsets[1].y,
757 &dst_rect.y0, &dst_rect.y1,
758 &push.y_off, &push.y_scale);
759
760 uint32_t dst_layer_count;
761 if (src_image->image_type == VK_IMAGE_TYPE_3D) {
762 uint32_t layer0, layer1;
763 compute_off_scale(src_extent.depth,
764 regions[r].srcOffsets[0].z,
765 regions[r].srcOffsets[1].z,
766 regions[r].dstOffsets[0].z,
767 regions[r].dstOffsets[1].z,
768 &layer0, &layer1,
769 &push.z_off, &push.z_scale);
770 dst_rect.layer = layer0;
771 dst_layer_count = layer1 - layer0;
772 } else {
773 assert(regions[r].srcSubresource.layerCount ==
774 regions[r].dstSubresource.layerCount);
775 dst_layer_count = regions[r].dstSubresource.layerCount;
776 push.arr_delta = regions[r].dstSubresource.baseArrayLayer -
777 regions[r].srcSubresource.baseArrayLayer;
778 }
779
780 do_blit(cmd, meta,
781 src_image, src_format, src_image_layout,
782 regions[r].srcSubresource,
783 dst_image, dst_format, dst_image_layout,
784 regions[r].dstSubresource,
785 sampler, &key, &push, &dst_rect, dst_layer_count);
786 }
787 }
788
789 void
vk_meta_blit_image2(struct vk_command_buffer * cmd,struct vk_meta_device * meta,const VkBlitImageInfo2 * blit)790 vk_meta_blit_image2(struct vk_command_buffer *cmd,
791 struct vk_meta_device *meta,
792 const VkBlitImageInfo2 *blit)
793 {
794 VK_FROM_HANDLE(vk_image, src_image, blit->srcImage);
795 VK_FROM_HANDLE(vk_image, dst_image, blit->dstImage);
796
797 vk_meta_blit_image(cmd, meta,
798 src_image, src_image->format, blit->srcImageLayout,
799 dst_image, dst_image->format, blit->dstImageLayout,
800 blit->regionCount, blit->pRegions, blit->filter);
801 }
802
803 void
vk_meta_resolve_image(struct vk_command_buffer * cmd,struct vk_meta_device * meta,struct vk_image * src_image,VkFormat src_format,VkImageLayout src_image_layout,struct vk_image * dst_image,VkFormat dst_format,VkImageLayout dst_image_layout,uint32_t region_count,const VkImageResolve2 * regions,VkResolveModeFlagBits resolve_mode,VkResolveModeFlagBits stencil_resolve_mode)804 vk_meta_resolve_image(struct vk_command_buffer *cmd,
805 struct vk_meta_device *meta,
806 struct vk_image *src_image,
807 VkFormat src_format,
808 VkImageLayout src_image_layout,
809 struct vk_image *dst_image,
810 VkFormat dst_format,
811 VkImageLayout dst_image_layout,
812 uint32_t region_count,
813 const VkImageResolve2 *regions,
814 VkResolveModeFlagBits resolve_mode,
815 VkResolveModeFlagBits stencil_resolve_mode)
816 {
817 struct vk_meta_blit_key key;
818 memset(&key, 0, sizeof(key));
819 key.key_type = VK_META_OBJECT_KEY_BLIT_PIPELINE;
820 key.dim = vk_image_sampler_dim(src_image);
821 key.src_samples = src_image->samples;
822 key.resolve_mode = resolve_mode;
823 key.stencil_resolve_mode = stencil_resolve_mode;
824 key.dst_format = dst_format;
825
826 for (uint32_t r = 0; r < region_count; r++) {
827 struct vk_meta_blit_push_data push = {
828 .x_off = regions[r].srcOffset.x - regions[r].dstOffset.x,
829 .y_off = regions[r].srcOffset.y - regions[r].dstOffset.y,
830 .x_scale = 1,
831 .y_scale = 1,
832 };
833 struct vk_meta_rect dst_rect = {
834 .x0 = regions[r].dstOffset.x,
835 .y0 = regions[r].dstOffset.y,
836 .x1 = regions[r].dstOffset.x + regions[r].extent.width,
837 .y1 = regions[r].dstOffset.y + regions[r].extent.height,
838 };
839
840 do_blit(cmd, meta,
841 src_image, src_format, src_image_layout,
842 regions[r].srcSubresource,
843 dst_image, dst_format, dst_image_layout,
844 regions[r].dstSubresource,
845 VK_NULL_HANDLE, &key, &push, &dst_rect,
846 regions[r].dstSubresource.layerCount);
847 }
848 }
849
850 void
vk_meta_resolve_image2(struct vk_command_buffer * cmd,struct vk_meta_device * meta,const VkResolveImageInfo2 * resolve)851 vk_meta_resolve_image2(struct vk_command_buffer *cmd,
852 struct vk_meta_device *meta,
853 const VkResolveImageInfo2 *resolve)
854 {
855 VK_FROM_HANDLE(vk_image, src_image, resolve->srcImage);
856 VK_FROM_HANDLE(vk_image, dst_image, resolve->dstImage);
857
858 VkResolveModeFlagBits resolve_mode = VK_RESOLVE_MODE_SAMPLE_ZERO_BIT;
859 if (vk_format_is_color(src_image->format) &&
860 !vk_format_is_int(src_image->format))
861 resolve_mode = VK_RESOLVE_MODE_AVERAGE_BIT;
862
863 vk_meta_resolve_image(cmd, meta,
864 src_image, src_image->format, resolve->srcImageLayout,
865 dst_image, dst_image->format, resolve->dstImageLayout,
866 resolve->regionCount, resolve->pRegions,
867 resolve_mode, VK_RESOLVE_MODE_SAMPLE_ZERO_BIT);
868 }
869
870 static void
vk_meta_resolve_attachment(struct vk_command_buffer * cmd,struct vk_meta_device * meta,struct vk_image_view * src_view,VkImageLayout src_image_layout,struct vk_image_view * dst_view,VkImageLayout dst_image_layout,VkImageAspectFlags resolve_aspects,VkResolveModeFlagBits resolve_mode,VkResolveModeFlagBits stencil_resolve_mode,VkRect2D area,uint32_t layer_count,uint32_t view_mask)871 vk_meta_resolve_attachment(struct vk_command_buffer *cmd,
872 struct vk_meta_device *meta,
873 struct vk_image_view *src_view,
874 VkImageLayout src_image_layout,
875 struct vk_image_view *dst_view,
876 VkImageLayout dst_image_layout,
877 VkImageAspectFlags resolve_aspects,
878 VkResolveModeFlagBits resolve_mode,
879 VkResolveModeFlagBits stencil_resolve_mode,
880 VkRect2D area, uint32_t layer_count,
881 uint32_t view_mask)
882 {
883 VkImageResolve2 region = {
884 .sType = VK_STRUCTURE_TYPE_IMAGE_RESOLVE_2,
885 .srcSubresource = {
886 .aspectMask = resolve_aspects,
887 .mipLevel = src_view->base_mip_level,
888 },
889 .srcOffset = { area.offset.x, area.offset.y, 0},
890 .dstSubresource = {
891 .aspectMask = resolve_aspects,
892 .mipLevel = dst_view->base_mip_level,
893 },
894 .dstOffset = { area.offset.x, area.offset.y, 0},
895 .extent = { area.extent.width, area.extent.height, 1},
896 };
897
898 if (view_mask) {
899 u_foreach_bit(v, view_mask) {
900 region.srcSubresource.baseArrayLayer = src_view->base_array_layer + v;
901 region.srcSubresource.layerCount = 1;
902 region.dstSubresource.baseArrayLayer = dst_view->base_array_layer + v;
903 region.dstSubresource.layerCount = 1;
904
905 vk_meta_resolve_image(cmd, meta,
906 src_view->image, src_view->format,
907 src_image_layout,
908 dst_view->image, dst_view->format,
909 dst_image_layout,
910 1, ®ion, resolve_mode, stencil_resolve_mode);
911 }
912 } else {
913 region.srcSubresource.baseArrayLayer = src_view->base_array_layer;
914 region.srcSubresource.layerCount = layer_count;
915 region.dstSubresource.baseArrayLayer = dst_view->base_array_layer;
916 region.dstSubresource.layerCount = layer_count;
917
918 vk_meta_resolve_image(cmd, meta,
919 src_view->image, src_view->format,
920 src_image_layout,
921 dst_view->image, dst_view->format,
922 dst_image_layout,
923 1, ®ion, resolve_mode, stencil_resolve_mode);
924 }
925 }
926
927 void
vk_meta_resolve_rendering(struct vk_command_buffer * cmd,struct vk_meta_device * meta,const VkRenderingInfo * pRenderingInfo)928 vk_meta_resolve_rendering(struct vk_command_buffer *cmd,
929 struct vk_meta_device *meta,
930 const VkRenderingInfo *pRenderingInfo)
931 {
932 for (uint32_t c = 0; c < pRenderingInfo->colorAttachmentCount; c++) {
933 const VkRenderingAttachmentInfo *att =
934 &pRenderingInfo->pColorAttachments[c];
935 if (att->resolveMode == VK_RESOLVE_MODE_NONE)
936 continue;
937
938 VK_FROM_HANDLE(vk_image_view, view, att->imageView);
939 VK_FROM_HANDLE(vk_image_view, res_view, att->resolveImageView);
940
941 vk_meta_resolve_attachment(cmd, meta, view, att->imageLayout,
942 res_view, att->resolveImageLayout,
943 VK_IMAGE_ASPECT_COLOR_BIT,
944 att->resolveMode, VK_RESOLVE_MODE_NONE,
945 pRenderingInfo->renderArea,
946 pRenderingInfo->layerCount,
947 pRenderingInfo->viewMask);
948 }
949
950 const VkRenderingAttachmentInfo *d_att = pRenderingInfo->pDepthAttachment;
951 if (d_att && d_att->resolveMode == VK_RESOLVE_MODE_NONE)
952 d_att = NULL;
953
954 const VkRenderingAttachmentInfo *s_att = pRenderingInfo->pStencilAttachment;
955 if (s_att && s_att->resolveMode == VK_RESOLVE_MODE_NONE)
956 s_att = NULL;
957
958 if (s_att != NULL || d_att != NULL) {
959 if (s_att != NULL && d_att != NULL &&
960 s_att->imageView == d_att->imageView &&
961 s_att->resolveImageView == d_att->resolveImageView) {
962 VK_FROM_HANDLE(vk_image_view, view, d_att->imageView);
963 VK_FROM_HANDLE(vk_image_view, res_view, d_att->resolveImageView);
964
965 vk_meta_resolve_attachment(cmd, meta, view, d_att->imageLayout,
966 res_view, d_att->resolveImageLayout,
967 VK_IMAGE_ASPECT_DEPTH_BIT |
968 VK_IMAGE_ASPECT_STENCIL_BIT,
969 d_att->resolveMode, s_att->resolveMode,
970 pRenderingInfo->renderArea,
971 pRenderingInfo->layerCount,
972 pRenderingInfo->viewMask);
973 } else {
974 if (d_att != NULL) {
975 VK_FROM_HANDLE(vk_image_view, view, d_att->imageView);
976 VK_FROM_HANDLE(vk_image_view, res_view, d_att->resolveImageView);
977
978 vk_meta_resolve_attachment(cmd, meta, view, d_att->imageLayout,
979 res_view, d_att->resolveImageLayout,
980 VK_IMAGE_ASPECT_DEPTH_BIT,
981 d_att->resolveMode, VK_RESOLVE_MODE_NONE,
982 pRenderingInfo->renderArea,
983 pRenderingInfo->layerCount,
984 pRenderingInfo->viewMask);
985 }
986
987 if (s_att != NULL) {
988 VK_FROM_HANDLE(vk_image_view, view, s_att->imageView);
989 VK_FROM_HANDLE(vk_image_view, res_view, s_att->resolveImageView);
990
991 vk_meta_resolve_attachment(cmd, meta, view, s_att->imageLayout,
992 res_view, s_att->resolveImageLayout,
993 VK_IMAGE_ASPECT_STENCIL_BIT,
994 VK_RESOLVE_MODE_NONE, s_att->resolveMode,
995 pRenderingInfo->renderArea,
996 pRenderingInfo->layerCount,
997 pRenderingInfo->viewMask);
998 }
999 }
1000 }
1001 }
1002