1from .common.codegen import CodeGen, VulkanWrapperGenerator
2from .common.vulkantypes import VulkanAPI, makeVulkanTypeSimple, iterateVulkanType, VulkanTypeInfo,\
3    VulkanType
4
5from .marshaling import VulkanMarshalingCodegen
6from .reservedmarshaling import VulkanReservedMarshalingCodegen
7from .transform import TransformCodegen
8
9from .wrapperdefs import API_PREFIX_MARSHAL
10from .wrapperdefs import API_PREFIX_RESERVEDUNMARSHAL
11from .wrapperdefs import MAX_PACKET_LENGTH
12from .wrapperdefs import VULKAN_STREAM_TYPE
13from .wrapperdefs import ROOT_TYPE_DEFAULT_VALUE
14from .wrapperdefs import RELAXED_APIS
15
16
17SKIPPED_DECODER_DELETES = [
18    "vkFreeDescriptorSets",
19]
20
21DELAYED_DECODER_DELETES = [
22    "vkDestroyPipelineLayout",
23]
24
25DELAYED_DECODER_DELETE_DICT_ENTRIES = [
26    "vkDestroyShaderModule",
27]
28
29global_state_prefix = "m_state->on_"
30
31decoder_decl_preamble = """
32
33namespace gfxstream {
34class IOStream;
35}  // namespace gfxstream
36
37namespace gfxstream {
38namespace vk {
39
40class VkDecoder {
41public:
42    VkDecoder();
43    ~VkDecoder();
44    void setForSnapshotLoad(bool forSnapshotLoad);
45    size_t decode(void* buf, size_t bufsize, IOStream* stream,
46                  const ProcessResources* processResources, const VkDecoderContext&);
47private:
48    class Impl;
49    std::unique_ptr<Impl> mImpl;
50};
51
52}  // namespace vk
53}  // namespace gfxstream
54
55"""
56
57decoder_impl_preamble ="""
58namespace gfxstream {
59namespace vk {
60
61using android::base::MetricEventBadPacketLength;
62using android::base::MetricEventDuplicateSequenceNum;
63
64class VkDecoder::Impl {
65public:
66    Impl() : m_logCalls(android::base::getEnvironmentVariable("ANDROID_EMU_VK_LOG_CALLS") == "1"),
67             m_vk(vkDispatch()),
68             m_state(VkDecoderGlobalState::get()),
69             m_vkStream(nullptr, m_state->getFeatures()),
70             m_vkMemReadingStream(nullptr, m_state->getFeatures()),
71             m_boxedHandleUnwrapMapping(m_state),
72             m_boxedHandleCreateMapping(m_state),
73             m_boxedHandleDestroyMapping(m_state),
74             m_boxedHandleUnwrapAndDeleteMapping(m_state),
75             m_boxedHandleUnwrapAndDeletePreserveBoxedMapping(m_state),
76             m_prevSeqno(std::nullopt),
77             m_queueSubmitWithCommandsEnabled(m_state->getFeatures().VulkanQueueSubmitWithCommands.enabled) {}
78    %s* stream() { return &m_vkStream; }
79    VulkanMemReadingStream* readStream() { return &m_vkMemReadingStream; }
80
81    void setForSnapshotLoad(bool forSnapshotLoad) {
82        m_forSnapshotLoad = forSnapshotLoad;
83    }
84
85    size_t decode(void* buf, size_t bufsize, IOStream* stream,
86                  const ProcessResources* processResources, const VkDecoderContext&);
87
88private:
89    bool m_logCalls;
90    bool m_forSnapshotLoad = false;
91    VulkanDispatch* m_vk;
92    VkDecoderGlobalState* m_state;
93    %s m_vkStream;
94    VulkanMemReadingStream m_vkMemReadingStream;
95    BoxedHandleUnwrapMapping m_boxedHandleUnwrapMapping;
96    BoxedHandleCreateMapping m_boxedHandleCreateMapping;
97    BoxedHandleDestroyMapping m_boxedHandleDestroyMapping;
98    BoxedHandleUnwrapAndDeleteMapping m_boxedHandleUnwrapAndDeleteMapping;
99    android::base::BumpPool m_pool;
100    BoxedHandleUnwrapAndDeletePreserveBoxedMapping m_boxedHandleUnwrapAndDeletePreserveBoxedMapping;
101    std::optional<uint32_t> m_prevSeqno;
102    bool m_queueSubmitWithCommandsEnabled = false;
103};
104
105VkDecoder::VkDecoder() :
106    mImpl(new VkDecoder::Impl()) { }
107
108VkDecoder::~VkDecoder() = default;
109
110void VkDecoder::setForSnapshotLoad(bool forSnapshotLoad) {
111    mImpl->setForSnapshotLoad(forSnapshotLoad);
112}
113
114size_t VkDecoder::decode(void* buf, size_t bufsize, IOStream* stream,
115                         const ProcessResources* processResources,
116                         const VkDecoderContext& context) {
117    return mImpl->decode(buf, bufsize, stream, processResources, context);
118}
119
120// VkDecoder::Impl::decode to follow
121""" % (VULKAN_STREAM_TYPE, VULKAN_STREAM_TYPE)
122
123decoder_impl_postamble = """
124
125}  // namespace vk
126}  // namespace gfxstream
127
128"""
129
130READ_STREAM = "vkReadStream"
131WRITE_STREAM = "vkStream"
132
133# Driver workarounds for APIs that don't work well multithreaded
134driver_workarounds_global_lock_apis = [ \
135    "vkCreatePipelineLayout",
136    "vkDestroyPipelineLayout",
137]
138
139def emit_param_decl_for_reading(param, cgen):
140    if param.staticArrExpr:
141        cgen.stmt(
142            cgen.makeRichCTypeDecl(param.getForNonConstAccess()))
143    else:
144        cgen.stmt(
145            cgen.makeRichCTypeDecl(param))
146
147def emit_unmarshal(typeInfo, param, cgen, output = False, destroy = False, noUnbox = False):
148    if destroy:
149        iterateVulkanType(typeInfo, param, VulkanReservedMarshalingCodegen(
150            cgen,
151            "host",
152            READ_STREAM,
153            ROOT_TYPE_DEFAULT_VALUE,
154            param.paramName,
155            "readStreamPtrPtr",
156            API_PREFIX_RESERVEDUNMARSHAL,
157            "",
158            direction="read",
159            dynAlloc=True))
160        lenAccess = cgen.generalLengthAccess(param)
161        lenAccessGuard = cgen.generalLengthAccessGuard(param)
162        if None == lenAccess or "1" == lenAccess:
163            cgen.stmt("boxed_%s_preserve = %s" % (param.paramName, param.paramName))
164            cgen.stmt("%s = unbox_%s(%s)" % (param.paramName, param.typeName, param.paramName))
165        else:
166            if lenAccessGuard is not None:
167                cgen.beginIf(lenAccessGuard)
168            cgen.beginFor("uint32_t i = 0", "i < %s" % lenAccess, "++i")
169            cgen.stmt("boxed_%s_preserve[i] = %s[i]" % (param.paramName, param.paramName))
170            cgen.stmt("((%s*)(%s))[i] = unbox_%s(%s[i])" % (param.typeName, param.paramName, param.typeName, param.paramName))
171            cgen.endFor()
172            if lenAccessGuard is not None:
173                cgen.endIf()
174    else:
175        if noUnbox:
176            cgen.line("// No unbox for %s" % (param.paramName))
177        iterateVulkanType(typeInfo, param, VulkanReservedMarshalingCodegen(
178            cgen,
179            "host",
180            READ_STREAM,
181            ROOT_TYPE_DEFAULT_VALUE,
182            param.paramName,
183            "readStreamPtrPtr",
184            API_PREFIX_RESERVEDUNMARSHAL,
185            "" if (output or noUnbox) else "unbox_",
186            direction="read",
187            dynAlloc=True))
188
189
190def emit_dispatch_unmarshal(typeInfo: VulkanTypeInfo, param: VulkanType, cgen, globalWrapped):
191    cgen.stmt("// Begin {} wrapped dispatchable handle unboxing for {}".format(
192        "global" if globalWrapped else "non",
193        param.paramName))
194
195    iterateVulkanType(typeInfo, param, VulkanReservedMarshalingCodegen(
196        cgen,
197        "host",
198        READ_STREAM,
199        ROOT_TYPE_DEFAULT_VALUE,
200        param.paramName,
201        "readStreamPtrPtr",
202        API_PREFIX_RESERVEDUNMARSHAL,
203        "",
204        direction="read",
205        dynAlloc=True))
206
207    if not globalWrapped:
208        cgen.stmt("auto unboxed_%s = unbox_%s(%s)" %
209                  (param.paramName, param.typeName, param.paramName))
210        cgen.stmt("auto vk = dispatch_%s(%s)" %
211                  (param.typeName, param.paramName))
212        cgen.stmt("// End manual dispatchable handle unboxing for %s" % param.paramName)
213
214
215def emit_transform(typeInfo, param, cgen, variant="tohost"):
216    res = iterateVulkanType(typeInfo, param, TransformCodegen(
217        cgen, param.paramName, "m_state", "transform_%s_" % variant, variant))
218    if not res:
219        cgen.stmt("(void)%s" % param.paramName)
220
221
222def emit_marshal(typeInfo, param, cgen, handleMapOverwrites=False):
223    iterateVulkanType(typeInfo, param, VulkanMarshalingCodegen(
224        cgen,
225        WRITE_STREAM,
226        ROOT_TYPE_DEFAULT_VALUE,
227        param.paramName,
228        API_PREFIX_MARSHAL,
229        direction="write",
230        handleMapOverwrites=handleMapOverwrites))
231
232
233class DecodingParameters(object):
234    def __init__(self, api: VulkanAPI):
235        self.params: list[VulkanType] = []
236        self.toRead: list[VulkanType] = []
237        self.toWrite: list[VulkanType] = []
238
239        for i, param in enumerate(api.parameters):
240            if i == 0 and param.isDispatchableHandleType():
241                param.dispatchHandle = True
242
243            if param.isNonDispatchableHandleType() and param.isCreatedBy(api):
244                param.nonDispatchableHandleCreate = True
245
246            if param.isNonDispatchableHandleType() and param.isDestroyedBy(api):
247                param.nonDispatchableHandleDestroy = True
248
249            if param.isDispatchableHandleType() and param.isCreatedBy(api):
250                param.dispatchableHandleCreate = True
251
252            if param.isDispatchableHandleType() and param.isDestroyedBy(api):
253                param.dispatchableHandleDestroy = True
254
255            self.toRead.append(param)
256
257            if param.possiblyOutput():
258                self.toWrite.append(param)
259
260            self.params.append(param)
261
262
263def emit_call_log(api, cgen):
264    decodingParams = DecodingParameters(api)
265    paramsToRead = decodingParams.toRead
266
267    cgen.beginIf("m_logCalls")
268    paramLogFormat = ""
269    paramLogArgs = []
270    for p in paramsToRead:
271        paramLogFormat += "0x%llx "
272    for p in paramsToRead:
273        paramLogArgs.append("(unsigned long long)%s" % (p.paramName))
274    cgen.stmt("fprintf(stderr, \"stream %%p: call %s %s\\n\", ioStream, %s)" % (api.name, paramLogFormat, ", ".join(paramLogArgs)))
275    cgen.endIf()
276
277def emit_decode_parameters(typeInfo: VulkanTypeInfo, api: VulkanAPI, cgen, globalWrapped=False):
278    decodingParams = DecodingParameters(api)
279
280    paramsToRead = decodingParams.toRead
281
282    for p in paramsToRead:
283        emit_param_decl_for_reading(p, cgen)
284
285    for i, p in enumerate(paramsToRead):
286        lenAccess = cgen.generalLengthAccess(p)
287
288        if p.dispatchHandle:
289            if api.name in DELAYED_DECODER_DELETE_DICT_ENTRIES:
290                emit_dispatch_unmarshal(typeInfo, p, cgen, False)
291            else:
292                emit_dispatch_unmarshal(typeInfo, p, cgen, globalWrapped)
293        else:
294            destroy = p.nonDispatchableHandleDestroy or p.dispatchableHandleDestroy
295            noUnbox = api.name in ["vkQueueFlushCommandsGOOGLE", "vkQueueFlushCommandsFromAuxMemoryGOOGLE"] and p.paramName == "commandBuffer"
296
297            if p.nonDispatchableHandleDestroy or p.dispatchableHandleDestroy:
298                destroy = True
299                cgen.stmt("// Begin manual non dispatchable handle destroy unboxing for %s" % p.paramName)
300                if None == lenAccess or "1" == lenAccess:
301                    cgen.stmt("%s boxed_%s_preserve" % (p.typeName, p.paramName))
302                else:
303                    cgen.stmt("%s* boxed_%s_preserve; %s->alloc((void**)&boxed_%s_preserve, %s * sizeof(%s))" % (p.typeName, p.paramName, READ_STREAM, p.paramName, lenAccess, p.typeName))
304
305            if p.possiblyOutput():
306                cgen.stmt("// Begin manual dispatchable handle unboxing for %s" % p.paramName)
307                cgen.stmt("%s->unsetHandleMapping()" % READ_STREAM)
308
309            emit_unmarshal(typeInfo, p, cgen, output = p.possiblyOutput(), destroy = destroy, noUnbox = noUnbox)
310
311    for p in paramsToRead:
312        emit_transform(typeInfo, p, cgen, variant="tohost")
313
314    emit_call_log(api, cgen)
315
316def emit_dispatch_call(api, cgen):
317
318    decodingParams = DecodingParameters(api)
319
320    customParams = []
321
322    delay = api.name in DELAYED_DECODER_DELETES
323
324    for i, p in enumerate(api.parameters):
325        customParam = p.paramName
326        if decodingParams.params[i].dispatchHandle:
327            customParam = "unboxed_%s" % p.paramName
328        customParams.append(customParam)
329
330    if delay:
331        cgen.line("std::function<void()> delayed_remove_callback = [vk, %s]() {" % ", ".join(customParams))
332
333    if api.name in driver_workarounds_global_lock_apis:
334        if delay:
335            cgen.stmt("auto state = VkDecoderGlobalState::get()")
336            cgen.stmt("// state already locked")
337        else:
338            cgen.stmt("m_state->lock()")
339
340    cgen.vkApiCall(api, customPrefix="vk->", customParameters=customParams, \
341        globalStatePrefix=global_state_prefix, checkForDeviceLost=True,
342        checkForOutOfMemory=True)
343
344    if api.name in driver_workarounds_global_lock_apis:
345        if not delay:
346            cgen.stmt("m_state->unlock()")
347        # for delayed remove, state is already locked, so we do not need to
348        # unlock
349
350    if delay:
351        cgen.line("};")
352
353def emit_global_state_wrapped_call(api, cgen, context):
354    if api.name in DELAYED_DECODER_DELETES:
355        print("Error: Cannot generate a global state wrapped call that is also a delayed delete (yet)");
356        raise
357
358    customParams = ["&m_pool"] + list(map(lambda p: p.paramName, api.parameters))
359    if context:
360        customParams += ["context"]
361    cgen.vkApiCall(api, customPrefix=global_state_prefix, \
362        customParameters=customParams, globalStatePrefix=global_state_prefix, \
363        checkForDeviceLost=True, checkForOutOfMemory=True)
364
365def emit_decode_parameters_writeback(typeInfo, api, cgen, autobox=True):
366    decodingParams = DecodingParameters(api)
367
368    paramsToWrite = decodingParams.toWrite
369
370    cgen.stmt("%s->unsetHandleMapping()" % WRITE_STREAM)
371
372    handleMapOverwrites = False
373
374    for p in paramsToWrite:
375        emit_transform(typeInfo, p, cgen, variant="fromhost")
376
377        handleMapOverwrites = False
378
379        if p.nonDispatchableHandleCreate or p.dispatchableHandleCreate:
380            handleMapOverwrites = True
381
382        if autobox and p.nonDispatchableHandleCreate:
383            cgen.stmt("// Begin auto non dispatchable handle create for %s" % p.paramName)
384            cgen.stmt("if (%s == VK_SUCCESS) %s->setHandleMapping(&m_boxedHandleCreateMapping)" % \
385                      (api.getRetVarExpr(), WRITE_STREAM))
386
387        if (not autobox) and p.nonDispatchableHandleCreate:
388            cgen.stmt("// Begin manual non dispatchable handle create for %s" % p.paramName)
389            cgen.stmt("%s->unsetHandleMapping()" % WRITE_STREAM)
390
391        emit_marshal(typeInfo, p, cgen, handleMapOverwrites=handleMapOverwrites)
392
393        if autobox and p.nonDispatchableHandleCreate:
394            cgen.stmt("// Begin auto non dispatchable handle create for %s" % p.paramName)
395            cgen.stmt("%s->setHandleMapping(&m_boxedHandleUnwrapMapping)" % WRITE_STREAM)
396
397        if (not autobox) and p.nonDispatchableHandleCreate:
398            cgen.stmt("// Begin manual non dispatchable handle create for %s" % p.paramName)
399            cgen.stmt("%s->setHandleMapping(&m_boxedHandleUnwrapMapping)" % WRITE_STREAM)
400
401def emit_decode_return_writeback(api, cgen):
402    retTypeName = api.getRetTypeExpr()
403    if retTypeName != "void":
404        retVar = api.getRetVarExpr()
405        cgen.stmt("%s->write(&%s, %s)" %
406            (WRITE_STREAM, retVar, cgen.sizeofExpr(api.retType)))
407
408def emit_decode_finish(api, cgen):
409    decodingParams = DecodingParameters(api)
410    retTypeName = api.getRetTypeExpr()
411    paramsToWrite = decodingParams.toWrite
412
413    if retTypeName != "void" or len(paramsToWrite) != 0:
414        cgen.stmt("%s->commitWrite()" % WRITE_STREAM)
415
416def emit_destroyed_handle_cleanup(api, cgen):
417    decodingParams = DecodingParameters(api)
418    paramsToRead = decodingParams.toRead
419
420    skipDelete = api.name in SKIPPED_DECODER_DELETES
421
422    if skipDelete:
423        cgen.line("// Skipping handle cleanup for %s" % api.name)
424        return
425
426    for p in paramsToRead:
427        if p.dispatchHandle:
428            pass
429        else:
430            lenAccess = cgen.generalLengthAccess(p)
431            lenAccessGuard = cgen.generalLengthAccess(p)
432            destroy = p.nonDispatchableHandleDestroy or p.dispatchableHandleDestroy
433            if destroy:
434                if None == lenAccess or "1" == lenAccess:
435                    if api.name in DELAYED_DECODER_DELETES:
436                        cgen.stmt("delayed_delete_%s(boxed_%s_preserve, unboxed_device, delayed_remove_callback)" % (p.typeName, p.paramName))
437                    elif api.name in DELAYED_DECODER_DELETE_DICT_ENTRIES:
438                        cgen.stmt("delayed_delete_%s(boxed_%s_preserve, unboxed_device, nullptr)" % (p.typeName, p.paramName))
439                    else:
440                        cgen.stmt("delete_%s(boxed_%s_preserve)" % (p.typeName, p.paramName))
441                else:
442                    if lenAccessGuard is not None:
443                        cgen.beginIf(lenAccessGuard)
444                    cgen.beginFor("uint32_t i = 0", "i < %s" % lenAccess, "++i")
445                    if api.name in DELAYED_DECODER_DELETES:
446                        cgen.stmt("delayed_delete_%s(boxed_%s_preserve[i], unboxed_device, delayed_remove_callback)" % (p.typeName, p.paramName))
447                    else:
448                        cgen.stmt("delete_%s(boxed_%s_preserve[i])" % (p.typeName, p.paramName))
449                    cgen.endFor()
450                    if lenAccessGuard is not None:
451                        cgen.endIf()
452
453def emit_pool_free(cgen):
454    cgen.stmt("%s->clearPool()" % READ_STREAM)
455
456def emit_seqno_incr(api, cgen):
457    cgen.stmt("if (m_queueSubmitWithCommandsEnabled) seqnoPtr->fetch_add(1, std::memory_order_seq_cst)")
458
459def emit_snapshot(typeInfo, api, cgen):
460
461    cgen.stmt("%s->setReadPos((uintptr_t)(*readStreamPtrPtr) - (uintptr_t)snapshotTraceBegin)" % READ_STREAM)
462    cgen.stmt("size_t snapshotTraceBytes = %s->endTrace()" % READ_STREAM)
463
464    additionalParams = [ \
465        makeVulkanTypeSimple(True, "uint8_t", 1, "snapshotTraceBegin"),
466        makeVulkanTypeSimple(False, "size_t", 0, "snapshotTraceBytes"),
467        makeVulkanTypeSimple(False, "android::base::BumpPool", 1, "&m_pool"),
468    ]
469
470    retTypeName = api.getRetTypeExpr()
471    if retTypeName != "void":
472        retVar = api.getRetVarExpr()
473        additionalParams.append(makeVulkanTypeSimple(False, retTypeName, 0, retVar))
474
475    paramsForSnapshot = []
476
477    decodingParams = DecodingParameters(api)
478
479    for p in decodingParams.toRead:
480        if p.nonDispatchableHandleDestroy or (not p.dispatchHandle and p.dispatchableHandleDestroy):
481            paramsForSnapshot.append(p.withModifiedName("boxed_%s_preserve" % p.paramName))
482        else:
483            paramsForSnapshot.append(p)
484
485    customParams = additionalParams + paramsForSnapshot
486
487    apiForSnapshot = \
488        api.withCustomReturnType(makeVulkanTypeSimple(False, "void", 0, "void")). \
489            withCustomParameters(customParams)
490
491    cgen.beginIf("m_state->snapshotsEnabled()")
492    cgen.vkApiCall(apiForSnapshot, customPrefix="m_state->snapshot()->")
493    cgen.endIf()
494
495def emit_decoding(typeInfo, api, cgen, globalWrapped=False, context=False):
496    isAcquire = api.name in RELAXED_APIS
497    emit_decode_parameters(typeInfo, api, cgen, globalWrapped)
498
499    if isAcquire:
500        emit_seqno_incr(api, cgen)
501
502    if globalWrapped:
503        emit_global_state_wrapped_call(api, cgen, context)
504    else:
505        emit_dispatch_call(api, cgen)
506
507    emit_decode_parameters_writeback(typeInfo, api, cgen, autobox=not globalWrapped)
508    emit_decode_return_writeback(api, cgen)
509    emit_decode_finish(api, cgen)
510    emit_snapshot(typeInfo, api, cgen)
511    emit_destroyed_handle_cleanup(api, cgen)
512    emit_pool_free(cgen)
513
514    if not isAcquire:
515        emit_seqno_incr(api, cgen)
516
517def emit_default_decoding(typeInfo, api, cgen):
518    emit_decoding(typeInfo, api, cgen)
519
520def emit_global_state_wrapped_decoding(typeInfo, api, cgen):
521    emit_decoding(typeInfo, api, cgen, globalWrapped=True)
522
523def emit_global_state_wrapped_decoding_with_context(typeInfo, api, cgen):
524    emit_decoding(typeInfo, api, cgen, globalWrapped=True, context=True)
525
526## Custom decoding definitions##################################################
527def decode_vkFlushMappedMemoryRanges(typeInfo: VulkanTypeInfo, api, cgen):
528    emit_decode_parameters(typeInfo, api, cgen)
529
530    cgen.beginIf("!m_state->usingDirectMapping()")
531    cgen.stmt("// This is to deal with a deficiency in the encoder,");
532    cgen.stmt("// where usingDirectMapping fails to set the proper packet size,");
533    cgen.stmt("// meaning we can read off the end of the packet.");
534    cgen.stmt("uint64_t sizeLeft = end - *readStreamPtrPtr")
535    cgen.beginFor("uint32_t i = 0", "i < memoryRangeCount", "++i")
536    cgen.beginIf("sizeLeft < sizeof(uint64_t)")
537    cgen.beginIf("m_prevSeqno")
538    cgen.stmt("m_prevSeqno = m_prevSeqno.value() - 1")
539    cgen.endIf()
540    cgen.stmt("return ptr - (unsigned char*)buf;")
541    cgen.endIf()
542    cgen.stmt("auto range = pMemoryRanges[i]")
543    cgen.stmt("auto memory = pMemoryRanges[i].memory")
544    cgen.stmt("auto size = pMemoryRanges[i].size")
545    cgen.stmt("auto offset = pMemoryRanges[i].offset")
546    cgen.stmt("uint64_t readStream = 0")
547    cgen.stmt("memcpy(&readStream, *readStreamPtrPtr, sizeof(uint64_t)); *readStreamPtrPtr += sizeof(uint64_t)")
548    cgen.stmt("sizeLeft -= sizeof(uint64_t)")
549    cgen.stmt("auto hostPtr = m_state->getMappedHostPointer(memory)")
550    cgen.stmt("if (!hostPtr && readStream > 0) GFXSTREAM_ABORT(::emugl::FatalError(::emugl::ABORT_REASON_OTHER))")
551    cgen.stmt("if (!hostPtr) continue")
552    cgen.beginIf("sizeLeft < readStream")
553    cgen.beginIf("m_prevSeqno")
554    cgen.stmt("m_prevSeqno = m_prevSeqno.value() - 1")
555    cgen.endIf()
556    cgen.stmt("return ptr - (unsigned char*)buf;")
557    cgen.endIf()
558    cgen.stmt("sizeLeft -= readStream")
559    cgen.stmt("uint8_t* targetRange = hostPtr + offset")
560    cgen.stmt("memcpy(targetRange, *readStreamPtrPtr, readStream); *readStreamPtrPtr += readStream")
561    cgen.stmt("packetLen += 8 + readStream")
562    cgen.endFor()
563    cgen.endIf()
564
565    emit_dispatch_call(api, cgen)
566    emit_decode_parameters_writeback(typeInfo, api, cgen)
567    emit_decode_return_writeback(api, cgen)
568    emit_decode_finish(api, cgen)
569    emit_snapshot(typeInfo, api, cgen);
570    emit_pool_free(cgen)
571    emit_seqno_incr(api, cgen)
572
573def decode_vkInvalidateMappedMemoryRanges(typeInfo, api, cgen):
574    emit_decode_parameters(typeInfo, api, cgen)
575    emit_dispatch_call(api, cgen)
576    emit_decode_parameters_writeback(typeInfo, api, cgen)
577    emit_decode_return_writeback(api, cgen)
578
579    cgen.beginIf("!m_state->usingDirectMapping()")
580    cgen.beginFor("uint32_t i = 0", "i < memoryRangeCount", "++i")
581    cgen.stmt("auto range = pMemoryRanges[i]")
582    cgen.stmt("auto memory = range.memory")
583    cgen.stmt("auto size = range.size")
584    cgen.stmt("auto offset = range.offset")
585    cgen.stmt("auto hostPtr = m_state->getMappedHostPointer(memory)")
586    cgen.stmt("auto actualSize = size == VK_WHOLE_SIZE ? m_state->getDeviceMemorySize(memory) : size")
587    cgen.stmt("uint64_t writeStream = 0")
588    cgen.stmt("if (!hostPtr) { %s->write(&writeStream, sizeof(uint64_t)); continue; }" % WRITE_STREAM)
589    cgen.stmt("uint8_t* targetRange = hostPtr + offset")
590    cgen.stmt("writeStream = actualSize")
591    cgen.stmt("%s->write(&writeStream, sizeof(uint64_t))" % WRITE_STREAM)
592    cgen.stmt("%s->write(targetRange, actualSize)" % WRITE_STREAM)
593    cgen.endFor()
594    cgen.endIf()
595
596    emit_decode_finish(api, cgen)
597    emit_snapshot(typeInfo, api, cgen);
598    emit_pool_free(cgen)
599    emit_seqno_incr(api, cgen)
600
601def decode_unsupported_api(typeInfo, api, cgen):
602    cgen.line(f"// Decoding {api.name} is not supported. This should not run.")
603    cgen.stmt(f"fprintf(stderr, \"stream %p: fatal: decoding unsupported API {api.name}\\n\", ioStream)");
604    cgen.stmt("__builtin_trap()")
605
606custom_decodes = {
607    "vkEnumerateInstanceVersion" : emit_global_state_wrapped_decoding,
608    "vkCreateInstance" : emit_global_state_wrapped_decoding,
609    "vkDestroyInstance" : emit_global_state_wrapped_decoding,
610    "vkEnumeratePhysicalDevices" : emit_global_state_wrapped_decoding,
611
612    "vkGetPhysicalDeviceFeatures" : emit_global_state_wrapped_decoding,
613    "vkGetPhysicalDeviceFeatures2" : emit_global_state_wrapped_decoding,
614    "vkGetPhysicalDeviceFeatures2KHR" : emit_global_state_wrapped_decoding,
615    "vkGetPhysicalDeviceFormatProperties" : emit_global_state_wrapped_decoding,
616    "vkGetPhysicalDeviceFormatProperties2" : emit_global_state_wrapped_decoding,
617    "vkGetPhysicalDeviceFormatProperties2KHR" : emit_global_state_wrapped_decoding,
618    "vkGetPhysicalDeviceImageFormatProperties" : emit_global_state_wrapped_decoding,
619    "vkGetPhysicalDeviceImageFormatProperties2" : emit_global_state_wrapped_decoding,
620    "vkGetPhysicalDeviceImageFormatProperties2KHR" : emit_global_state_wrapped_decoding,
621    "vkGetPhysicalDeviceProperties" : emit_global_state_wrapped_decoding,
622    "vkGetPhysicalDeviceProperties2" : emit_global_state_wrapped_decoding,
623    "vkGetPhysicalDeviceProperties2KHR" : emit_global_state_wrapped_decoding,
624
625    "vkGetPhysicalDeviceMemoryProperties" : emit_global_state_wrapped_decoding,
626    "vkGetPhysicalDeviceMemoryProperties2" : emit_global_state_wrapped_decoding,
627    "vkGetPhysicalDeviceMemoryProperties2KHR" : emit_global_state_wrapped_decoding,
628
629    "vkGetPhysicalDeviceExternalSemaphoreProperties" : emit_global_state_wrapped_decoding,
630    "vkGetPhysicalDeviceExternalSemaphorePropertiesKHR" : emit_global_state_wrapped_decoding,
631
632    "vkEnumerateDeviceExtensionProperties" : emit_global_state_wrapped_decoding,
633
634    "vkCreateBuffer" : emit_global_state_wrapped_decoding,
635    "vkDestroyBuffer" : emit_global_state_wrapped_decoding,
636
637    "vkBindBufferMemory" : emit_global_state_wrapped_decoding,
638    "vkBindBufferMemory2" : emit_global_state_wrapped_decoding,
639    "vkBindBufferMemory2KHR" : emit_global_state_wrapped_decoding,
640
641    "vkCreateDevice" : emit_global_state_wrapped_decoding,
642    "vkGetDeviceQueue" : emit_global_state_wrapped_decoding,
643    "vkDestroyDevice" : emit_global_state_wrapped_decoding,
644
645    "vkGetDeviceQueue2" : emit_global_state_wrapped_decoding,
646
647    "vkBindImageMemory" : emit_global_state_wrapped_decoding,
648    "vkBindImageMemory2" : emit_global_state_wrapped_decoding,
649    "vkBindImageMemory2KHR" : emit_global_state_wrapped_decoding,
650
651    "vkCreateImage" : emit_global_state_wrapped_decoding,
652    "vkCreateImageView" : emit_global_state_wrapped_decoding,
653    "vkCreateSampler" : emit_global_state_wrapped_decoding,
654    "vkDestroyImage" : emit_global_state_wrapped_decoding,
655    "vkDestroyImageView" : emit_global_state_wrapped_decoding,
656    "vkDestroySampler" : emit_global_state_wrapped_decoding,
657    "vkCmdCopyBufferToImage" : emit_global_state_wrapped_decoding_with_context,
658    "vkCmdCopyImage" : emit_global_state_wrapped_decoding,
659    "vkCmdCopyImageToBuffer" : emit_global_state_wrapped_decoding,
660    "vkCmdCopyBufferToImage2" : emit_global_state_wrapped_decoding_with_context,
661    "vkCmdCopyImage2" : emit_global_state_wrapped_decoding,
662    "vkCmdCopyImageToBuffer2" : emit_global_state_wrapped_decoding,
663    "vkGetImageMemoryRequirements" : emit_global_state_wrapped_decoding,
664    "vkGetImageMemoryRequirements2" : emit_global_state_wrapped_decoding,
665    "vkGetImageMemoryRequirements2KHR" : emit_global_state_wrapped_decoding,
666    "vkGetBufferMemoryRequirements" : emit_global_state_wrapped_decoding,
667    "vkGetBufferMemoryRequirements2": emit_global_state_wrapped_decoding,
668    "vkGetBufferMemoryRequirements2KHR": emit_global_state_wrapped_decoding,
669
670    "vkCreateDescriptorSetLayout" : emit_global_state_wrapped_decoding,
671    "vkDestroyDescriptorSetLayout" : emit_global_state_wrapped_decoding,
672    "vkCreateDescriptorPool" : emit_global_state_wrapped_decoding,
673    "vkDestroyDescriptorPool" : emit_global_state_wrapped_decoding,
674    "vkResetDescriptorPool" : emit_global_state_wrapped_decoding,
675    "vkAllocateDescriptorSets" : emit_global_state_wrapped_decoding,
676    "vkFreeDescriptorSets" : emit_global_state_wrapped_decoding,
677
678    "vkUpdateDescriptorSets" : emit_global_state_wrapped_decoding,
679
680    "vkCreateShaderModule": emit_global_state_wrapped_decoding,
681    "vkDestroyShaderModule": emit_global_state_wrapped_decoding,
682    "vkCreatePipelineCache": emit_global_state_wrapped_decoding,
683    "vkDestroyPipelineCache": emit_global_state_wrapped_decoding,
684    "vkCreateGraphicsPipelines": emit_global_state_wrapped_decoding,
685    "vkDestroyPipeline": emit_global_state_wrapped_decoding,
686
687    "vkAllocateMemory" : emit_global_state_wrapped_decoding,
688    "vkFreeMemory" : emit_global_state_wrapped_decoding,
689    "vkMapMemory" : emit_global_state_wrapped_decoding,
690    "vkUnmapMemory" : emit_global_state_wrapped_decoding,
691    "vkFlushMappedMemoryRanges" : decode_vkFlushMappedMemoryRanges,
692    "vkInvalidateMappedMemoryRanges" : decode_vkInvalidateMappedMemoryRanges,
693
694    "vkAllocateCommandBuffers" : emit_global_state_wrapped_decoding,
695    "vkCmdExecuteCommands" : emit_global_state_wrapped_decoding,
696    "vkQueueSubmit" : emit_global_state_wrapped_decoding,
697    "vkQueueSubmit2" : emit_global_state_wrapped_decoding,
698    "vkQueueWaitIdle" : emit_global_state_wrapped_decoding,
699    "vkBeginCommandBuffer" : emit_global_state_wrapped_decoding_with_context,
700    "vkEndCommandBuffer" : emit_global_state_wrapped_decoding_with_context,
701    "vkResetCommandBuffer" : emit_global_state_wrapped_decoding,
702    "vkFreeCommandBuffers" : emit_global_state_wrapped_decoding,
703    "vkCreateCommandPool" : emit_global_state_wrapped_decoding,
704    "vkDestroyCommandPool" : emit_global_state_wrapped_decoding,
705    "vkResetCommandPool" : emit_global_state_wrapped_decoding,
706    "vkCmdPipelineBarrier" : emit_global_state_wrapped_decoding,
707    "vkCmdBindPipeline" : emit_global_state_wrapped_decoding,
708    "vkCmdBindDescriptorSets" : emit_global_state_wrapped_decoding,
709
710    "vkCreateRenderPass" : emit_global_state_wrapped_decoding,
711    "vkCreateRenderPass2" : emit_global_state_wrapped_decoding,
712    "vkCreateRenderPass2KHR" : emit_global_state_wrapped_decoding,
713    "vkDestroyRenderPass" : emit_global_state_wrapped_decoding,
714    "vkCreateFramebuffer" : emit_global_state_wrapped_decoding,
715    "vkDestroyFramebuffer" : emit_global_state_wrapped_decoding,
716    "vkDestroyFramebuffer" : emit_global_state_wrapped_decoding,
717    "vkCmdBeginRenderPass" : emit_global_state_wrapped_decoding,
718    "vkCmdBeginRenderPass2" : emit_global_state_wrapped_decoding,
719    "vkCmdBeginRenderPass2KHR" : emit_global_state_wrapped_decoding,
720
721    "vkCreateSamplerYcbcrConversion": emit_global_state_wrapped_decoding,
722    "vkDestroySamplerYcbcrConversion": emit_global_state_wrapped_decoding,
723
724    # VK_ANDROID_native_buffer
725    "vkGetSwapchainGrallocUsageANDROID" : emit_global_state_wrapped_decoding,
726    "vkGetSwapchainGrallocUsage2ANDROID" : emit_global_state_wrapped_decoding,
727    "vkAcquireImageANDROID" : emit_global_state_wrapped_decoding,
728    "vkQueueSignalReleaseImageANDROID" : emit_global_state_wrapped_decoding,
729
730    "vkCreateSemaphore" : emit_global_state_wrapped_decoding,
731    "vkGetSemaphoreFdKHR" : emit_global_state_wrapped_decoding,
732    "vkImportSemaphoreFdKHR" : emit_global_state_wrapped_decoding,
733    "vkDestroySemaphore" : emit_global_state_wrapped_decoding,
734
735    "vkCreateFence" : emit_global_state_wrapped_decoding,
736    "vkResetFences" : emit_global_state_wrapped_decoding,
737    "vkDestroyFence" : emit_global_state_wrapped_decoding,
738
739    # VK_GOOGLE_gfxstream
740    "vkFreeMemorySyncGOOGLE" : emit_global_state_wrapped_decoding,
741    "vkMapMemoryIntoAddressSpaceGOOGLE" : emit_global_state_wrapped_decoding,
742    "vkGetMemoryHostAddressInfoGOOGLE" : emit_global_state_wrapped_decoding,
743    "vkGetBlobGOOGLE" : emit_global_state_wrapped_decoding,
744
745    # Descriptor update templates
746    "vkCreateDescriptorUpdateTemplate" : emit_global_state_wrapped_decoding,
747    "vkCreateDescriptorUpdateTemplateKHR" : emit_global_state_wrapped_decoding,
748    "vkDestroyDescriptorUpdateTemplate" : emit_global_state_wrapped_decoding,
749    "vkDestroyDescriptorUpdateTemplateKHR" : emit_global_state_wrapped_decoding,
750    "vkUpdateDescriptorSetWithTemplateSizedGOOGLE" : emit_global_state_wrapped_decoding,
751    "vkUpdateDescriptorSetWithTemplateSized2GOOGLE" : emit_global_state_wrapped_decoding,
752
753    # VK_GOOGLE_gfxstream
754    "vkBeginCommandBufferAsyncGOOGLE" : emit_global_state_wrapped_decoding_with_context,
755    "vkEndCommandBufferAsyncGOOGLE" : emit_global_state_wrapped_decoding_with_context,
756    "vkResetCommandBufferAsyncGOOGLE" : emit_global_state_wrapped_decoding,
757    "vkCommandBufferHostSyncGOOGLE" : emit_global_state_wrapped_decoding,
758    "vkCreateImageWithRequirementsGOOGLE" : emit_global_state_wrapped_decoding,
759    "vkCreateBufferWithRequirementsGOOGLE" : emit_global_state_wrapped_decoding,
760    "vkQueueHostSyncGOOGLE" : emit_global_state_wrapped_decoding,
761    "vkQueueSubmitAsyncGOOGLE" : emit_global_state_wrapped_decoding,
762    "vkQueueSubmitAsync2GOOGLE" : emit_global_state_wrapped_decoding,
763    "vkQueueWaitIdleAsyncGOOGLE" : emit_global_state_wrapped_decoding,
764    "vkQueueBindSparseAsyncGOOGLE" : emit_global_state_wrapped_decoding,
765    "vkGetLinearImageLayoutGOOGLE" : emit_global_state_wrapped_decoding,
766    "vkGetLinearImageLayout2GOOGLE" : emit_global_state_wrapped_decoding,
767    "vkQueueFlushCommandsGOOGLE" : emit_global_state_wrapped_decoding_with_context,
768    "vkQueueFlushCommandsFromAuxMemoryGOOGLE" : emit_global_state_wrapped_decoding_with_context,
769    "vkQueueCommitDescriptorSetUpdatesGOOGLE" : emit_global_state_wrapped_decoding,
770    "vkCollectDescriptorPoolIdsGOOGLE" : emit_global_state_wrapped_decoding,
771    "vkQueueSignalReleaseImageANDROIDAsyncGOOGLE" : emit_global_state_wrapped_decoding,
772
773    "vkQueueBindSparse" : emit_global_state_wrapped_decoding,
774
775    # VK_KHR_xcb_surface
776    "vkCreateXcbSurfaceKHR": decode_unsupported_api,
777    "vkGetPhysicalDeviceXcbPresentationSupportKHR": decode_unsupported_api,
778
779    # VK_EXT_metal_surface
780    "vkCreateMetalSurfaceEXT": decode_unsupported_api,
781
782    # VK_KHR_sampler_ycbcr_conversion
783    "vkCreateSamplerYcbcrConversionKHR": emit_global_state_wrapped_decoding,
784    "vkDestroySamplerYcbcrConversionKHR": emit_global_state_wrapped_decoding,
785
786    #VK_KHR_copy_commands2
787    "vkCmdCopyBufferToImage2KHR" : emit_global_state_wrapped_decoding_with_context,
788    "vkCmdCopyImage2KHR" : emit_global_state_wrapped_decoding,
789    "vkCmdCopyImageToBuffer2KHR" : emit_global_state_wrapped_decoding,
790}
791
792class VulkanDecoder(VulkanWrapperGenerator):
793    def __init__(self, module, typeInfo):
794        VulkanWrapperGenerator.__init__(self, module, typeInfo)
795        self.typeInfo: VulkanTypeInfo = typeInfo
796        self.cgen = CodeGen()
797
798    def onBegin(self,):
799        self.module.appendImpl(
800            "#define MAX_PACKET_LENGTH %s\n" % MAX_PACKET_LENGTH)
801        self.module.appendHeader(decoder_decl_preamble)
802        self.module.appendImpl(decoder_impl_preamble)
803
804        self.module.appendImpl(
805            """
806size_t VkDecoder::Impl::decode(void* buf, size_t len, IOStream* ioStream,
807                               const ProcessResources* processResources,
808                               const VkDecoderContext& context)
809""")
810
811        self.cgen.beginBlock() # function body
812
813        self.cgen.stmt("const char* processName = context.processName")
814        self.cgen.stmt("auto& gfx_logger = *context.gfxApiLogger")
815        self.cgen.stmt("auto* healthMonitor = context.healthMonitor")
816        self.cgen.stmt("auto& metricsLogger = *context.metricsLogger")
817        self.cgen.stmt("if (len < 8) return 0")
818        self.cgen.stmt("unsigned char *ptr = (unsigned char *)buf")
819        self.cgen.stmt("const unsigned char* const end = (const unsigned char*)buf + len")
820
821        self.cgen.beginIf("m_forSnapshotLoad")
822        self.cgen.stmt("ptr += m_state->setCreatedHandlesForSnapshotLoad(ptr)");
823        self.cgen.endIf()
824        self.cgen.line("while (end - ptr >= 8)")
825        self.cgen.beginBlock() # while loop
826
827        self.cgen.stmt("uint32_t opcode = *(uint32_t *)ptr")
828        self.cgen.stmt("uint32_t packetLen = *(uint32_t *)(ptr + 4)")
829        self.cgen.line("""
830        // packetLen should be at least 8 (op code and packet length) and should not be excessively large
831        if (packetLen < 8 || packetLen > MAX_PACKET_LENGTH) {
832            WARN("Bad packet length %d detected, decode may fail", packetLen);
833            metricsLogger.logMetricEvent(MetricEventBadPacketLength{ .len = packetLen });
834        }
835        """)
836        self.cgen.stmt("if (end - ptr < packetLen) return ptr - (unsigned char*)buf")
837        self.cgen.stmt("gfx_logger.record(ptr, std::min(size_t(packetLen + 8), size_t(end - ptr)))")
838
839        self.cgen.stmt("stream()->setStream(ioStream)")
840        self.cgen.stmt("VulkanStream* %s = stream()" % WRITE_STREAM)
841        self.cgen.stmt("VulkanMemReadingStream* %s = readStream()" % READ_STREAM)
842        self.cgen.stmt("%s->setBuf((uint8_t*)(ptr + 8))" % READ_STREAM)
843        self.cgen.stmt("uint8_t* readStreamPtr = %s->getBuf(); uint8_t** readStreamPtrPtr = &readStreamPtr" % READ_STREAM)
844        self.cgen.stmt("uint8_t* snapshotTraceBegin = %s->beginTrace()" % READ_STREAM)
845        self.cgen.stmt("%s->setHandleMapping(&m_boxedHandleUnwrapMapping)" % READ_STREAM)
846        self.cgen.line("""
847        std::unique_ptr<EventHangMetadata::HangAnnotations> executionData =
848            std::make_unique<EventHangMetadata::HangAnnotations>();
849        if (healthMonitor) {
850            executionData->insert(
851                 {{"packet_length", std::to_string(packetLen)},
852                 {"opcode", std::to_string(opcode)}});
853            if (processName) {
854                executionData->insert(
855                    {{"renderthread_guest_process", std::string(processName)}});
856            }
857            if (m_prevSeqno) {
858                executionData->insert({{"previous_seqno", std::to_string(m_prevSeqno.value())}});
859            }
860        }
861
862        std::atomic<uint32_t>* seqnoPtr = processResources ?
863                processResources->getSequenceNumberPtr() : nullptr;
864
865        if (m_queueSubmitWithCommandsEnabled && ((opcode >= OP_vkFirst && opcode < OP_vkLast) || (opcode >= OP_vkFirst_old && opcode < OP_vkLast_old))) {
866            uint32_t seqno;
867            memcpy(&seqno, *readStreamPtrPtr, sizeof(uint32_t)); *readStreamPtrPtr += sizeof(uint32_t);
868            if (healthMonitor) executionData->insert({{"seqno", std::to_string(seqno)}});
869            if (m_prevSeqno  && seqno == m_prevSeqno.value()) {
870                WARN(
871                    "Seqno %d is the same as previously processed on thread %d. It might be a "
872                    "duplicate command.",
873                    seqno, getCurrentThreadId());
874                metricsLogger.logMetricEvent(MetricEventDuplicateSequenceNum{ .opcode = opcode });
875            }
876            if (seqnoPtr && !m_forSnapshotLoad) {
877                {
878                    auto seqnoWatchdog =
879                        WATCHDOG_BUILDER(healthMonitor,
880                                         "RenderThread seqno loop")
881                            .setHangType(EventHangMetadata::HangType::kRenderThread)
882                            .setAnnotations(std::make_unique<EventHangMetadata::HangAnnotations>(*executionData))
883                            /* Data gathered if this hangs*/
884                            .setOnHangCallback([=]() {
885                                auto annotations = std::make_unique<EventHangMetadata::HangAnnotations>();
886                                annotations->insert({{"seqnoPtr", std::to_string(seqnoPtr->load(std::memory_order_seq_cst))}});
887                                return annotations;
888                            })
889                            .build();
890                    while ((seqno - seqnoPtr->load(std::memory_order_seq_cst) != 1)) {
891                        #if (defined(_MSC_VER) && (defined(_M_IX86) || defined(_M_X64)))
892                        _mm_pause();
893                        #elif (defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__)))
894                        __asm__ __volatile__("pause;");
895                        #endif
896                    }
897                    m_prevSeqno = seqno;
898                }
899            }
900        }
901        """)
902
903        self.cgen.line("""
904        gfx_logger.recordCommandExecution();
905        """)
906
907        self.cgen.line("""
908        auto executionWatchdog =
909            WATCHDOG_BUILDER(healthMonitor, "RenderThread VkDecoder command execution")
910                .setHangType(EventHangMetadata::HangType::kRenderThread)
911                .setAnnotations(std::move(executionData))
912                .build();
913        """)
914
915        self.cgen.stmt("auto vk = m_vk")
916
917        self.cgen.line("switch (opcode)")
918        self.cgen.beginBlock()  # switch stmt
919
920        self.module.appendImpl(self.cgen.swapCode())
921
922    def onGenCmd(self, cmdinfo, name, alias):
923        typeInfo = self.typeInfo
924        cgen = self.cgen
925        api: VulkanAPI = typeInfo.apis[name]
926
927        cgen.line("case OP_%s:" % name)
928        cgen.beginBlock()
929        cgen.stmt("android::base::beginTrace(\"%s decode\")" % name)
930
931        if api.name in custom_decodes.keys():
932            custom_decodes[api.name](typeInfo, api, cgen)
933        else:
934            emit_default_decoding(typeInfo, api, cgen)
935
936        cgen.stmt("android::base::endTrace()")
937        cgen.stmt("break")
938        cgen.endBlock()
939        self.module.appendImpl(self.cgen.swapCode())
940
941    def onEnd(self,):
942        self.cgen.line("default:")
943        self.cgen.beginBlock()
944        self.cgen.stmt("m_pool.freeAll()")
945        self.cgen.stmt("return ptr - (unsigned char *)buf")
946        self.cgen.endBlock()
947
948        self.cgen.endBlock() # switch stmt
949
950        self.cgen.stmt("ptr += packetLen")
951        self.cgen.endBlock() # while loop
952
953        self.cgen.beginIf("m_forSnapshotLoad")
954        self.cgen.stmt("m_state->clearCreatedHandlesForSnapshotLoad()");
955        self.cgen.endIf()
956
957        self.cgen.stmt("m_pool.freeAll()")
958        self.cgen.stmt("return ptr - (unsigned char*)buf;")
959        self.cgen.endBlock() # function body
960        self.module.appendImpl(self.cgen.swapCode())
961        self.module.appendImpl(decoder_impl_postamble)
962