1/*
2 * Copyright (C) 2012 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "asm_support_x86_64.S"
18
19#define MANAGED_ARGS_SAVE_SIZE /*xmm0-xmm7*/ 8 * 8 + /*padding*/ 8 + /* GPR args */ 6 * 8
20
21MACRO0(SAVE_MANAGED_ARGS_INCREASE_FRAME)
22    // Return address is on the stack.
23    PUSH_ARG r9
24    PUSH_ARG r8
25    PUSH_ARG rcx
26    PUSH_ARG rdx
27    PUSH_ARG rsi
28    PUSH_ARG rdi
29    INCREASE_FRAME (/*FPRs*/ 8 * 8 + /*padding*/ 8)
30    movsd %xmm0, 0(%rsp)
31    movsd %xmm1, 8(%rsp)
32    movsd %xmm2, 16(%rsp)
33    movsd %xmm3, 24(%rsp)
34    movsd %xmm4, 32(%rsp)
35    movsd %xmm5, 40(%rsp)
36    movsd %xmm6, 48(%rsp)
37    movsd %xmm7, 56(%rsp)
38END_MACRO
39
40MACRO0(RESTORE_MANAGED_ARGS_DECREASE_FRAME)
41    movsd 0(%rsp), %xmm0
42    movsd 8(%rsp), %xmm1
43    movsd 16(%rsp), %xmm2
44    movsd 24(%rsp), %xmm3
45    movsd 32(%rsp), %xmm4
46    movsd 40(%rsp), %xmm5
47    movsd 48(%rsp), %xmm6
48    movsd 56(%rsp), %xmm7
49    DECREASE_FRAME (/*FPRs*/ 8 * 8 + /*padding*/ 8)
50    POP_ARG rdi
51    POP_ARG rsi
52    POP_ARG rdx
53    POP_ARG rcx
54    POP_ARG r8
55    POP_ARG r9
56END_MACRO
57
58MACRO3(JNI_SAVE_MANAGED_ARGS_TRAMPOLINE, name, cxx_name, arg1)
59DEFINE_FUNCTION \name
60    // Note: Managed callee-save registers have been saved by the JNI stub.
61    // Save register args RDI, RSI, RDX, RCX, R8, R9, mmx0-mmx7 and align stack.
62    SAVE_MANAGED_ARGS_INCREASE_FRAME
63    // Call `cxx_name()`.
64    .ifnc \arg1, none
65        mov REG_VAR(arg1), %rdi     // Pass arg1.
66    .endif
67    call CALLVAR(cxx_name)          // Call cxx_name(...).
68    // Restore register args RDI, RSI, RDX, RCX, R8, R9, mmx0-mmx7 and return.
69    RESTORE_MANAGED_ARGS_DECREASE_FRAME
70    ret
71END_FUNCTION \name
72END_MACRO
73
74MACRO4(JNI_SAVE_RETURN_VALUE_TRAMPOLINE, name, cxx_name, arg1, arg2)
75DEFINE_FUNCTION \name
76    // Save return registers and return address.
77    PUSH_ARG rax
78    INCREASE_FRAME /*mmx0*/ 8 + /*padding*/ 8
79    movsd %xmm0, 0(%rsp)
80    // Call `cxx_name()`.
81    mov REG_VAR(arg1), %rdi         // Pass arg1.
82    .ifnc \arg2, none
83        mov REG_VAR(arg2), %rsi     // Pass arg2.
84    .endif
85    call CALLVAR(cxx_name)          // Call cxx_name(...).
86    // Restore return registers and return.
87    movsd 0(%rsp), %xmm0
88    DECREASE_FRAME /*mmx0*/ 8 + /*padding*/ 8
89    POP_ARG rax
90    ret
91END_FUNCTION \name
92END_MACRO
93
94    /*
95     * Jni dlsym lookup stub.
96     */
97DEFINE_FUNCTION art_jni_dlsym_lookup_stub
98    // Save callee and GPR args.
99    PUSH_ARG r9   // Arg.
100    PUSH_ARG r8   // Arg.
101    PUSH_ARG rdi  // Arg. (JniEnv for normal and @FastNative)
102    PUSH_ARG rsi  // Arg.
103    PUSH_ARG rdx  // Arg.
104    PUSH_ARG rcx  // Arg.
105    // Create space for FPR args, plus padding for alignment
106    INCREASE_FRAME 72
107    // Save FPRs.
108    movq %xmm0, 0(%rsp)
109    movq %xmm1, 8(%rsp)
110    movq %xmm2, 16(%rsp)
111    movq %xmm3, 24(%rsp)
112    movq %xmm4, 32(%rsp)
113    movq %xmm5, 40(%rsp)
114    movq %xmm6, 48(%rsp)
115    movq %xmm7, 56(%rsp)
116    // prepare call
117    movq %gs:THREAD_SELF_OFFSET, %rdi      // RDI := Thread::Current()
118    // Call artFindNativeMethod() for normal native and artFindNativeMethodRunnable()
119    // for @FastNative or @CriticalNative.
120    movq THREAD_TOP_QUICK_FRAME_OFFSET(%rdi), %rax   // uintptr_t tagged_quick_frame
121    andq LITERAL(TAGGED_JNI_SP_MASK_TOGGLED64), %rax // ArtMethod** sp
122    movq (%rax), %rax                                // ArtMethod* method
123    testl LITERAL(ACCESS_FLAGS_METHOD_IS_FAST_NATIVE | ACCESS_FLAGS_METHOD_IS_CRITICAL_NATIVE), \
124          ART_METHOD_ACCESS_FLAGS_OFFSET(%rax)
125    jne .Llookup_stub_fast_or_critical_native
126    call SYMBOL(artFindNativeMethod)  // (Thread*)
127    jmp .Llookup_stub_continue
128.Llookup_stub_fast_or_critical_native:
129    call SYMBOL(artFindNativeMethodRunnable)  // (Thread*)
130.Llookup_stub_continue:
131    // restore arguments
132    movq 0(%rsp), %xmm0
133    movq 8(%rsp), %xmm1
134    movq 16(%rsp), %xmm2
135    movq 24(%rsp), %xmm3
136    movq 32(%rsp), %xmm4
137    movq 40(%rsp), %xmm5
138    movq 48(%rsp), %xmm6
139    movq 56(%rsp), %xmm7
140    DECREASE_FRAME 72
141    POP_ARG rcx  // Arg.
142    POP_ARG rdx  // Arg.
143    POP_ARG rsi  // Arg.
144    POP_ARG rdi  // Arg. (JniEnv for normal and @FastNative)
145    POP_ARG r8   // Arg.
146    POP_ARG r9   // Arg.
147    testq %rax, %rax              // check if returned method code is null
148    jz .Lno_native_code_found     // if null, jump to return to handle
149    jmp *%rax                     // otherwise, tail call to intended method
150.Lno_native_code_found:
151    ret
152END_FUNCTION art_jni_dlsym_lookup_stub
153
154    /*
155     * Jni dlsym lookup stub for @CriticalNative.
156     */
157DEFINE_FUNCTION art_jni_dlsym_lookup_critical_stub
158    // The hidden arg holding the tagged method (bit 0 set means GenericJNI) is RAX.
159    // For Generic JNI we already have a managed frame, so we reuse the art_jni_dlsym_lookup_stub.
160    testq LITERAL(1), %rax
161    jnz art_jni_dlsym_lookup_stub
162
163    // Save GPR args and method.
164    PUSH_ARG r9
165    PUSH_ARG r8
166    PUSH_ARG rdi
167    PUSH_ARG rsi
168    PUSH_ARG rdx
169    PUSH_ARG rcx
170    PUSH_ARG rax
171    // Create space for FPR args.
172    INCREASE_FRAME 8 * 8
173    // Save FPRs.
174    movq %xmm0, 0(%rsp)
175    movq %xmm1, 8(%rsp)
176    movq %xmm2, 16(%rsp)
177    movq %xmm3, 24(%rsp)
178    movq %xmm4, 32(%rsp)
179    movq %xmm5, 40(%rsp)
180    movq %xmm6, 48(%rsp)
181    movq %xmm7, 56(%rsp)
182    // Note: It's the caller's responsibility to preserve xmm12-xmm15 as the tail call
183    // to native shall always risk clobbering those.
184
185    // Call artCriticalNativeFrameSize(method, caller_pc).
186    movq %rax, %rdi       // Pass the method from hidden arg.
187    movq 120(%rsp), %rsi  // Pass caller PC.
188    call SYMBOL(artCriticalNativeFrameSize)
189
190    // Restore registers.
191    movq 0(%rsp), %xmm0
192    movq 8(%rsp), %xmm1
193    movq 16(%rsp), %xmm2
194    movq 24(%rsp), %xmm3
195    movq 32(%rsp), %xmm4
196    movq 40(%rsp), %xmm5
197    movq 48(%rsp), %xmm6
198    movq 56(%rsp), %xmm7
199    DECREASE_FRAME 8 * 8
200    POP_ARG r10  // Restore method to R10.
201    POP_ARG rcx
202    POP_ARG rdx
203    POP_ARG rsi
204    POP_ARG rdi
205    POP_ARG r8
206    POP_ARG r9
207
208    // Load caller PC to R11 and redefine return PC for CFI.
209    movq (%rsp), %r11
210    CFI_REGISTER(%rip, %r11)
211
212    // Reserve space for a SaveRefsAndArgs managed frame, either for the actual runtime
213    // method or for a GenericJNI frame which is similar but has a native method and a tag.
214    INCREASE_FRAME FRAME_SIZE_SAVE_REFS_AND_ARGS - __SIZEOF_POINTER__
215
216    // Calculate the number of QWORDs to move.
217    shrq LITERAL(3), %rax
218    jz .Lcritical_skip_copy_args
219
220    // Save RDI, RSI, RCX so that we can use them for moving stack args.
221    PUSH_ARG rdi
222    PUSH_ARG rsi
223    PUSH_ARG rcx
224
225    // Move the stack args.
226    movq %rax, %rcx
227    leaq 3 * __SIZEOF_POINTER__(%rsp), %rdi
228    leaq FRAME_SIZE_SAVE_REFS_AND_ARGS(%rdi), %rsi
229    rep movsq
230
231    // Restore RDI, RSI, RCX.
232    POP_ARG rcx
233    POP_ARG rsi
234    POP_ARG rdi
235
236.Lcritical_skip_copy_args:
237    // Calculate the base address of the managed frame.
238    leaq (%rsp, %rax, 8), %rax
239
240    // Spill registers for the SaveRefsAndArgs frame above the stack args.
241    // Note that the runtime shall not examine the args here, otherwise we would have to
242    // move them in registers and stack to account for the difference between managed and
243    // native ABIs. Do not update CFI while we hold the frame address in RAX and the values
244    // in registers are unchanged.
245    movq %r15, 192(%rax)
246    movq %r14, 184(%rax)
247    movq %r13, 176(%rax)
248    movq %r12, 168(%rax)
249    movq %r9, 160(%rax)
250    movq %r8, 152(%rax)
251    movq %rsi, 144(%rax)
252    movq %rbp, 136(%rax)
253    movq %rbx, 128(%rax)
254    movq %rdx, 120(%rax)
255    movq %rcx, 112(%rax)
256    movq %xmm0, 16(%rax)
257    movq %xmm1, 24(%rax)
258    movq %xmm2, 32(%rax)
259    movq %xmm3, 40(%rax)
260    movq %xmm4, 48(%rax)
261    movq %xmm5, 56(%rax)
262    movq %xmm6, 64(%rax)
263    movq %xmm7, 72(%rax)
264    // Skip managed ABI callee-saves xmm12-xmm15.
265
266    // Move the managed frame address to native callee-save register RBP and update CFI.
267    movq %rax, %rbp
268    CFI_EXPRESSION_BREG CFI_REG(r15), CFI_REG(rbp), 192
269    CFI_EXPRESSION_BREG CFI_REG(r14), CFI_REG(rbp), 184
270    CFI_EXPRESSION_BREG CFI_REG(r13), CFI_REG(rbp), 176
271    CFI_EXPRESSION_BREG CFI_REG(r12), CFI_REG(rbp), 168
272    // Skip args r9, r8, rsi.
273    CFI_EXPRESSION_BREG CFI_REG(rbp), CFI_REG(rbp), 136
274    CFI_EXPRESSION_BREG CFI_REG(rbx), CFI_REG(rbp), 128
275    // Skip args rdx, rcx.
276    // Skip args xmm0-xmm7.
277
278    leaq 1(%rbp), %rax            // Prepare managed SP tagged for a GenericJNI frame.
279    testl LITERAL(ACCESS_FLAGS_METHOD_IS_NATIVE), ART_METHOD_ACCESS_FLAGS_OFFSET(%r10)
280    jnz .Lcritical_skip_prepare_runtime_method
281
282    // Save the return PC for managed stack walk.
283    // (When coming from a compiled stub, the correct return PC is already there.)
284    movq %r11, FRAME_SIZE_SAVE_REFS_AND_ARGS - __SIZEOF_POINTER__(%rbp)
285
286    // Replace the target method with the SaveRefsAndArgs runtime method.
287    LOAD_RUNTIME_INSTANCE r10
288    movq RUNTIME_SAVE_REFS_AND_ARGS_METHOD_OFFSET(%r10), %r10
289
290    movq %rbp, %rax               // Prepare untagged managed SP for the runtime method.
291
292.Lcritical_skip_prepare_runtime_method:
293    // Store the method on the bottom of the managed frame.
294    movq %r10, (%rbp)
295
296    // Place (maybe tagged) managed SP in Thread::Current()->top_quick_frame.
297    movq %rax, %gs:THREAD_TOP_QUICK_FRAME_OFFSET
298
299    // Save our return PC in the padding.
300    movq %r11, __SIZEOF_POINTER__(%rbp)
301    CFI_EXPRESSION_BREG CFI_REG(rip), CFI_REG(rbp), __SIZEOF_POINTER__
302
303    // Preserve the native arg register RDI in callee-save register RBX which was saved above.
304    movq %rdi, %rbx
305
306    // Call artFindNativeMethodRunnable()
307    movq %gs:THREAD_SELF_OFFSET, %rdi  // pass Thread::Current()
308    call SYMBOL(artFindNativeMethodRunnable)  // (Thread*)
309
310    // Check for exception.
311    test %rax, %rax
312    CFI_REMEMBER_STATE
313    jz .Lcritical_deliver_exception
314
315    // Restore the native arg register RDI.
316    movq %rbx, %rdi
317
318    // Remember our return PC in R11.
319    movq __SIZEOF_POINTER__(%rbp), %r11
320    CFI_REGISTER(%rip, %r11)
321
322    // Remember the frame base address in r10 but do not redefine CFI.
323    movq %rbp, %r10
324
325    // Restore the frame. We shall not need the method anymore.
326    movq 16(%rbp), %xmm0
327    movq 24(%rbp), %xmm1
328    movq 32(%rbp), %xmm2
329    movq 40(%rbp), %xmm3
330    movq 48(%rbp), %xmm4
331    movq 56(%rbp), %xmm5
332    movq 64(%rbp), %xmm6
333    movq 72(%rbp), %xmm7
334    // Skip managed callee-saves xmm12-xmm15.
335    movq 112(%rbp), %rcx
336    movq 120(%rbp), %rdx
337    RESTORE_REG_BASE rbp, rbx, 128
338    // Delay restoring RBP as it's the managed frame base.
339    movq 144(%rbp), %rsi
340    movq 152(%rbp), %r8
341    movq 160(%rbp), %r9
342    RESTORE_REG_BASE rbp, r12, 168
343    RESTORE_REG_BASE rbp, r13, 176
344    RESTORE_REG_BASE rbp, r14, 184
345    RESTORE_REG_BASE rbp, r15, 192
346    // Restore RBP last.
347    RESTORE_REG_BASE rbp, rbp, 136
348
349    cmp %r10, %rsp
350    je .Lcritical_skip_copy_args_back
351
352    // Save RDI, RSI, RCX so that we can use them for moving stack args.
353    PUSH_ARG rdi
354    PUSH_ARG rsi
355    PUSH_ARG rcx
356
357    // Calculate the number of QWORDs to move.
358    leaq -3 * __SIZEOF_POINTER__(%r10), %rcx
359    subq %rsp, %rcx
360    shrq LITERAL(3), %rcx
361
362    // Move the stack args.
363    leaq -__SIZEOF_POINTER__(%r10), %rsi
364    leaq FRAME_SIZE_SAVE_REFS_AND_ARGS - __SIZEOF_POINTER__(%r10), %rdi
365    std
366    rep movsq
367    cld
368
369    // Restore RDI, RSI, RCX.
370    POP_ARG rcx
371    POP_ARG rsi
372    POP_ARG rdi
373
374.Lcritical_skip_copy_args_back:
375    // Remove the frame reservation.
376    DECREASE_FRAME FRAME_SIZE_SAVE_REFS_AND_ARGS - __SIZEOF_POINTER__
377
378    // Store our return PC.
379    movq %r11, (%rsp)
380    CFI_REL_OFFSET(%rip, 0)
381
382    // Do the tail call.
383    jmp *%rax
384
385.Lcritical_deliver_exception:
386    CFI_RESTORE_STATE_AND_DEF_CFA %rbp, FRAME_SIZE_SAVE_REFS_AND_ARGS
387    DELIVER_PENDING_EXCEPTION_FRAME_READY
388END_FUNCTION art_jni_dlsym_lookup_critical_stub
389
390    /*
391     * Read barrier for the method's declaring class needed by JNI stub for static methods.
392     * (We're using a pointer to the declaring class in `ArtMethod` as `jclass`.)
393     */
394JNI_SAVE_MANAGED_ARGS_TRAMPOLINE art_jni_read_barrier, artJniReadBarrier, none
395
396    /*
397     * Trampoline to `artJniMethodStart()` that preserves all managed arguments.
398     */
399JNI_SAVE_MANAGED_ARGS_TRAMPOLINE art_jni_method_start, artJniMethodStart, gs:THREAD_SELF_OFFSET
400
401    /*
402     * Trampoline to `artJniMethodEntryHook` that preserves all managed arguments.
403     */
404JNI_SAVE_MANAGED_ARGS_TRAMPOLINE \
405    art_jni_method_entry_hook, artJniMethodEntryHook, gs:THREAD_SELF_OFFSET
406
407    /*
408     * Trampoline to `artJniMonitoredMethodStart()` that preserves all managed arguments.
409     */
410JNI_SAVE_MANAGED_ARGS_TRAMPOLINE \
411    art_jni_monitored_method_start, artJniMonitoredMethodStart, gs:THREAD_SELF_OFFSET
412
413    /*
414     * Trampoline to `artJniMethodEnd()` that preserves all return registers.
415     */
416JNI_SAVE_RETURN_VALUE_TRAMPOLINE art_jni_method_end, artJniMethodEnd, gs:THREAD_SELF_OFFSET, none
417
418    /*
419     * Trampoline to `artJniMonitoredMethodEnd()` that preserves all return registers.
420     */
421JNI_SAVE_RETURN_VALUE_TRAMPOLINE \
422    art_jni_monitored_method_end, artJniMonitoredMethodEnd, gs:THREAD_SELF_OFFSET, none
423
424    /*
425     * Entry from JNI stub that tries to lock the object in a fast path and
426     * calls `artLockObjectFromCode()` (the same as for managed code) for the
427     * difficult cases, may block for GC.
428     * Custom calling convention:
429     *     RBX holds the non-null object to lock.
430     *     Callee-save registers have been saved and can be used as temporaries (except RBX).
431     *     All argument registers need to be preserved.
432     */
433DEFINE_FUNCTION art_jni_lock_object
434    LOCK_OBJECT_FAST_PATH rbx, ebp, art_jni_lock_object_no_inline
435END_FUNCTION art_jni_lock_object
436
437    /*
438     * Entry from JNI stub that calls `artLockObjectFromCode()`
439     * (the same as for managed code), may block for GC.
440     * Custom calling convention:
441     *     RBX holds the non-null object to lock.
442     *     Callee-save registers have been saved and can be used as temporaries (except RBX).
443     *     All argument registers need to be preserved.
444     */
445DEFINE_FUNCTION art_jni_lock_object_no_inline
446    // This is also the slow path for art_jni_lock_object.
447    // Save register args RDI, RSI, RDX, RCX, R8, R9, mmx0-mmx7 and align stack.
448    SAVE_MANAGED_ARGS_INCREASE_FRAME
449    // Call `artLockObjectFromCode()`
450    movq %rbx, %rdi                       // Pass the object to lock.
451    movq %gs:THREAD_SELF_OFFSET, %rsi     // Pass Thread::Current().
452    call SYMBOL(artLockObjectFromCode)    // (object, Thread*)
453    // Check result.
454    testl %eax, %eax
455    jnz   1f
456    // Restore register args RDI, RSI, RDX, RCX, R8, R9, mmx0-mmx7 and return.
457    RESTORE_MANAGED_ARGS_DECREASE_FRAME
458    ret
459    .cfi_adjust_cfa_offset MANAGED_ARGS_SAVE_SIZE
4601:
461    // All args are irrelevant when throwing an exception. Remove the spill area.
462    DECREASE_FRAME MANAGED_ARGS_SAVE_SIZE
463    // Rely on the JNI transition frame constructed in the JNI stub.
464    movq %gs:THREAD_SELF_OFFSET, %rdi     // Pass Thread::Current().
465    jmp  SYMBOL(artDeliverPendingExceptionFromCode)  // (Thread*); tail call.
466END_FUNCTION art_jni_lock_object_no_inline
467
468    /*
469     * Entry from JNI stub that tries to unlock the object in a fast path and calls
470     * `artJniUnlockObject()` for the difficult cases. Note that failure to unlock
471     * is fatal, so we do not need to check for exceptions in the slow path.
472     * Custom calling convention:
473     *     RBX holds the non-null object to unlock.
474     *     Callee-save registers have been saved and can be used as temporaries (except RBX).
475     *     Return registers RAX and mmx0 need to be preserved.
476     */
477DEFINE_FUNCTION art_jni_unlock_object
478    movq %rax, %r12                       // Preserve RAX in a different register.
479    UNLOCK_OBJECT_FAST_PATH rbx, ebp, /*saved_rax*/ r12, .Lunlock_object_jni_slow
480
481 .Lunlock_object_jni_slow:
482    movq %r12, %rax                       // Restore RAX.
483    jmp  SYMBOL(art_jni_unlock_object_no_inline)
484END_FUNCTION art_jni_unlock_object
485
486    /*
487     * Entry from JNI stub that calls `artJniUnlockObject()`. Note that failure to
488     * unlock is fatal, so we do not need to check for exceptions.
489     * Custom calling convention:
490     *     RBX holds the non-null object to unlock.
491     *     Callee-save registers have been saved and can be used as temporaries (except RBX).
492     *     Return registers RAX and mmx0 need to be preserved.
493     */
494    // This is also the slow path for art_jni_unlock_object.
495JNI_SAVE_RETURN_VALUE_TRAMPOLINE \
496    art_jni_unlock_object_no_inline, artJniUnlockObject, rbx, gs:THREAD_SELF_OFFSET
497