1%def header(): 2/* 3 * Copyright (C) 2020 The Android Open Source Project 4 * 5 * Licensed under the Apache License, Version 2.0 (the "License"); 6 * you may not use this file except in compliance with the License. 7 * You may obtain a copy of the License at 8 * 9 * http://www.apache.org/licenses/LICENSE-2.0 10 * 11 * Unless required by applicable law or agreed to in writing, software 12 * distributed under the License is distributed on an "AS IS" BASIS, 13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 * See the License for the specific language governing permissions and 15 * limitations under the License. 16 */ 17 18/* 19 * This is a #include, not a %include, because we want the C pre-processor 20 * to expand the macros into assembler assignment statements. 21 */ 22#include "asm_support.h" 23#include "arch/arm64/asm_support_arm64.S" 24 25/** 26 * ARM64 Runtime register usage conventions. 27 * 28 * r0 : w0 is 32-bit return register and x0 is 64-bit. 29 * r0-r7 : Argument registers. 30 * r8-r15 : Caller save registers (used as temporary registers). 31 * r16-r17: Also known as ip0-ip1, respectively. Used as scratch registers by 32 * the linker, by the trampolines and other stubs (the compiler uses 33 * these as temporary registers). 34 * r18 : Reserved for platform (SCS, shadow call stack) 35 * r19 : Pointer to thread-local storage. 36 * r20-r29: Callee save registers. 37 * r30 : (lr) is reserved (the link register). 38 * rsp : (sp) is reserved (the stack pointer). 39 * rzr : (zr) is reserved (the zero register). 40 * 41 * Floating-point registers 42 * v0-v31 43 * 44 * v0 : s0 is return register for singles (32-bit) and d0 for doubles (64-bit). 45 * This is analogous to the C/C++ (hard-float) calling convention. 46 * v0-v7 : Floating-point argument registers in both Dalvik and C/C++ conventions. 47 * Also used as temporary and codegen scratch registers. 48 * 49 * v0-v7 and v16-v31 : Caller save registers (used as temporary registers). 50 * v8-v15 : bottom 64-bits preserved across C calls (d8-d15 are preserved). 51 * 52 * v16-v31: Used as codegen temp/scratch. 53 * v8-v15 : Can be used for promotion. 54 * 55 * Must maintain 16-byte stack alignment. 56 * 57 * Nterp notes: 58 * 59 * The following registers have fixed assignments: 60 * 61 * reg nick purpose 62 * x19 xSELF self (Thread) pointer 63 * x20 wMR marking register 64 * x21 xSUSPEND suspend check register 65 * x29 xFP interpreted frame pointer, used for accessing locals and args 66 * x22 xPC interpreted program counter, used for fetching instructions 67 * x23 xINST first 16-bit code unit of current instruction 68 * x24 xIBASE interpreted instruction base pointer, used for computed goto 69 * x25 xREFS base of object references of dex registers. 70 * x16 ip scratch reg 71 * x17 ip2 scratch reg (used by macros) 72 * 73 * Macros are provided for common operations. They MUST NOT alter unspecified registers or 74 * condition codes. 75*/ 76 77/* single-purpose registers, given names for clarity */ 78#define CFI_DEX 22 // DWARF register number of the register holding dex-pc (xPC). 79#define CFI_TMP 0 // DWARF register number of the first argument register (r0). 80#define xPC x22 81#define xINST x23 82#define wINST w23 83#define xIBASE x24 84#define xREFS x25 85#define CFI_REFS 25 86#define ip x16 87#define ip2 x17 88#define wip w16 89#define wip2 w17 90 91// To avoid putting ifdefs arond the use of wMR, make sure it's defined. 92// IsNterpSupported returns false for configurations that don't have wMR (typically CMS). 93#ifndef wMR 94#define wMR w20 95#endif 96 97// Temporary registers while setting up a frame. 98#define xNEW_FP x26 99#define xNEW_REFS x27 100#define CFI_NEW_REFS 27 101 102// +8 for the ArtMethod of the caller. 103#define OFFSET_TO_FIRST_ARGUMENT_IN_STACK (CALLEE_SAVES_SIZE + 8) 104 105/* 106 * Fetch the next instruction from xPC into wINST. Does not advance xPC. 107 */ 108.macro FETCH_INST 109 ldrh wINST, [xPC] 110.endm 111 112/* 113 * Fetch the next instruction from the specified offset. Advances xPC 114 * to point to the next instruction. "count" is in 16-bit code units. 115 * 116 * Because of the limited size of immediate constants on ARM, this is only 117 * suitable for small forward movements (i.e. don't try to implement "goto" 118 * with this). 119 * 120 * This must come AFTER anything that can throw an exception, or the 121 * exception catch may miss. (This also implies that it must come after 122 * EXPORT_PC.) 123 */ 124.macro FETCH_ADVANCE_INST count 125 ldrh wINST, [xPC, #((\count)*2)]! 126.endm 127 128/* 129 * Similar to FETCH_ADVANCE_INST, but does not update xPC. Used to load 130 * xINST ahead of possible exception point. Be sure to manually advance xPC 131 * later. 132 */ 133.macro PREFETCH_INST count 134 ldrh wINST, [xPC, #((\count)*2)] 135.endm 136 137/* Advance xPC by some number of code units. */ 138.macro ADVANCE count 139 add xPC, xPC, #((\count)*2) 140.endm 141 142/* 143 * Fetch a half-word code unit from an offset past the current PC. The 144 * "count" value is in 16-bit code units. Does not advance xPC. 145 * 146 * The "_S" variant works the same but treats the value as signed. 147 */ 148.macro FETCH reg, count 149 ldrh \reg, [xPC, #((\count)*2)] 150.endm 151 152.macro FETCH_S reg, count 153 ldrsh \reg, [xPC, #((\count)*2)] 154.endm 155 156/* 157 * Fetch one byte from an offset past the current PC. Pass in the same 158 * "count" as you would for FETCH, and an additional 0/1 indicating which 159 * byte of the halfword you want (lo/hi). 160 */ 161.macro FETCH_B reg, count, byte 162 ldrb \reg, [xPC, #((\count)*2+(\byte))] 163.endm 164 165/* 166 * Put the instruction's opcode field into the specified register. 167 */ 168.macro GET_INST_OPCODE reg 169 and \reg, xINST, #255 170.endm 171 172/* 173 * Begin executing the opcode in _reg. Clobbers reg 174 */ 175 176.macro GOTO_OPCODE reg 177 add \reg, xIBASE, \reg, lsl #${handler_size_bits} 178 br \reg 179.endm 180 181/* 182 * Get/set the 32-bit value from a Dalvik register. 183 */ 184.macro GET_VREG reg, vreg 185 ldr \reg, [xFP, \vreg, uxtw #2] 186.endm 187.macro GET_VREG_OBJECT reg, vreg 188 ldr \reg, [xREFS, \vreg, uxtw #2] 189.endm 190.macro SET_VREG reg, vreg 191 str \reg, [xFP, \vreg, uxtw #2] 192 str wzr, [xREFS, \vreg, uxtw #2] 193.endm 194.macro SET_VREG_OBJECT reg, vreg 195 str \reg, [xFP, \vreg, uxtw #2] 196 str \reg, [xREFS, \vreg, uxtw #2] 197.endm 198.macro SET_VREG_FLOAT reg, vreg 199 str \reg, [xFP, \vreg, uxtw #2] 200 str wzr, [xREFS, \vreg, uxtw #2] 201.endm 202 203/* 204 * Get/set the 64-bit value from a Dalvik register. 205 */ 206.macro LOAD_SCALED_VREG_MASK scaled_mask_reg, unscaled_mask 207 mov \scaled_mask_reg, \unscaled_mask << 2 208.endm 209.macro EXTRACT_SCALED_VREG scaled_vreg, scaled_mask_reg, src_reg, lsb 210 .if \lsb < 2 211 and \scaled_vreg, \scaled_mask_reg, \src_reg, lsl #(2 - \lsb) 212 .else 213 and \scaled_vreg, \scaled_mask_reg, \src_reg, lsr #(\lsb - 2) 214 .endif 215.endm 216.macro GET_VREG_WIDE_PRESCALED reg, scaled_vreg 217 ldr \reg, [xFP, \scaled_vreg, uxtw] 218.endm 219.macro GET_VREG_WIDE reg, vreg 220 lsl wip2, \vreg, #2 221 GET_VREG_WIDE_PRESCALED \reg, wip2 222.endm 223.macro SET_VREG_WIDE_PRESCALED reg, scaled_vreg 224 str \reg, [xFP, \scaled_vreg, uxtw] 225 str xzr, [xREFS, \scaled_vreg, uxtw] 226.endm 227.macro SET_VREG_WIDE reg, vreg 228 lsl wip2, \vreg, #2 229 SET_VREG_WIDE_PRESCALED \reg, wip2 230.endm 231.macro GET_VREG_DOUBLE_PRESCALED reg, scaled_vreg 232 GET_VREG_WIDE_PRESCALED \reg, \scaled_vreg 233.endm 234.macro GET_VREG_DOUBLE reg, vreg 235 GET_VREG_WIDE \reg, \vreg 236.endm 237.macro SET_VREG_DOUBLE reg, vreg 238 SET_VREG_WIDE \reg, \vreg 239.endm 240 241/* 242 * Get the 32-bit value from a Dalvik register and sign-extend to 64-bit. 243 * Used to avoid an extra instruction in int-to-long. 244 */ 245.macro GET_VREG_S reg, vreg 246 ldrsw \reg, [xFP, \vreg, uxtw #2] 247.endm 248 249// An assembly entry for nterp. 250.macro OAT_ENTRY name 251 .type \name, #function 252 .hidden \name 253 .global \name 254 .balign 16 255\name: 256.endm 257 258.macro SIZE name 259 .size \name, .-\name 260.endm 261 262.macro NAME_START name 263 .type \name, #function 264 .hidden \name // Hide this as a global symbol, so we do not incur plt calls. 265 .global \name 266 /* Cache alignment for function entry */ 267 .balign 16 268\name: 269.endm 270 271.macro NAME_END name 272 SIZE \name 273.endm 274 275// Macro for defining entrypoints into runtime. We don't need to save registers 276// (we're not holding references there), but there is no 277// kDontSave runtime method. So just use the kSaveRefsOnly runtime method. 278.macro NTERP_TRAMPOLINE name, helper 279ENTRY \name 280 SETUP_SAVE_REFS_ONLY_FRAME 281 bl \helper 282 RESTORE_SAVE_REFS_ONLY_FRAME 283 REFRESH_MARKING_REGISTER 284 ldr xIP0, [xSELF, # THREAD_EXCEPTION_OFFSET] // Get exception field. 285 cbnz xIP0, nterp_deliver_pending_exception 286 ret 287END \name 288.endm 289 290.macro CLEAR_STATIC_VOLATILE_MARKER reg 291 and \reg, \reg, #-2 292.endm 293 294.macro CLEAR_INSTANCE_VOLATILE_MARKER reg 295 neg \reg, \reg 296.endm 297 298.macro EXPORT_PC 299 str xPC, [xREFS, #-16] 300.endm 301 302.macro BRANCH 303 add xPC, xPC, wINST, sxtw #1 // update xPC 304 // Update method counter and do a suspend check if the branch is negative or zero. 305 cmp wINST, #0 306 b.le 2f 3071: 308 FETCH wINST, 0 // load wINST 309 GET_INST_OPCODE ip // extract opcode from wINST 310 GOTO_OPCODE ip // jump to next instruction 3112: 312 ldr x0, [sp] 313 ldrh w2, [x0, #ART_METHOD_HOTNESS_COUNT_OFFSET] 314#if (NTERP_HOTNESS_VALUE != 0) 315#error Expected 0 for hotness value 316#endif 317 // If the counter is at zero, handle this in the runtime. 318 cbz w2, NterpHandleHotnessOverflow 319 add x2, x2, #-1 320 strh w2, [x0, #ART_METHOD_HOTNESS_COUNT_OFFSET] 321 DO_SUSPEND_CHECK continue_label=1b 322 b 1b 323.endm 324 325// Uses x12, x13, and x14 as temporaries. 326.macro FETCH_CODE_ITEM_INFO code_item, registers, outs, ins, load_ins 327 tbz \code_item, #0, 4f 328 and \code_item, \code_item, #-2 // Remove the extra bit that marks it's a compact dex file 329 ldrh w13, [\code_item, #COMPACT_CODE_ITEM_FIELDS_OFFSET] 330 ubfx \registers, w13, #COMPACT_CODE_ITEM_REGISTERS_SIZE_SHIFT, #4 331 ubfx \outs, w13, #COMPACT_CODE_ITEM_OUTS_SIZE_SHIFT, #4 332 .if \load_ins 333 ubfx \ins, w13, #COMPACT_CODE_ITEM_INS_SIZE_SHIFT, #4 334 .else 335 ubfx w14, w13, #COMPACT_CODE_ITEM_INS_SIZE_SHIFT, #4 336 add \registers, \registers, w14 337 .endif 338 ldrh w13, [\code_item, #COMPACT_CODE_ITEM_FLAGS_OFFSET] 339 tst w13, #COMPACT_CODE_ITEM_REGISTERS_INS_OUTS_FLAGS 340 b.eq 3f 341 sub x14, \code_item, #4 342 tst w13, #COMPACT_CODE_ITEM_INSNS_FLAG 343 csel x14, x14, \code_item, ne 344 345 tbz w13, #COMPACT_CODE_ITEM_REGISTERS_BIT, 1f 346 ldrh w12, [x14, #-2]! 347 add \registers, \registers, w12 3481: 349 tbz w13, #COMPACT_CODE_ITEM_INS_BIT, 2f 350 ldrh w12, [x14, #-2]! 351 .if \load_ins 352 add \ins, \ins, w12 353 .else 354 add \registers, \registers, w12 355 .endif 3562: 357 tbz w13, #COMPACT_CODE_ITEM_OUTS_BIT, 3f 358 ldrh w12, [x14, #-2]! 359 add \outs, \outs, w12 3603: 361 .if \load_ins 362 add \registers, \registers, \ins 363 .endif 364 add \code_item, \code_item, #COMPACT_CODE_ITEM_INSNS_OFFSET 365 b 5f 3664: 367 // Fetch dex register size. 368 ldrh \registers, [\code_item, #CODE_ITEM_REGISTERS_SIZE_OFFSET] 369 // Fetch outs size. 370 ldrh \outs, [\code_item, #CODE_ITEM_OUTS_SIZE_OFFSET] 371 .if \load_ins 372 ldrh \ins, [\code_item, #CODE_ITEM_INS_SIZE_OFFSET] 373 .endif 374 add \code_item, \code_item, #CODE_ITEM_INSNS_OFFSET 3755: 376.endm 377 378.macro TEST_IF_MARKING label 379 cbnz wMR, \label 380.endm 381 382// Setup the stack to start executing the method. Expects: 383// - x0 to contain the ArtMethod 384// 385// Outputs 386// - ip contains the dex registers size 387// - x28 contains the old stack pointer. 388// - \code_item is replaced with a pointer to the instructions 389// - if load_ins is 1, w15 contains the ins 390// 391// Uses ip, ip2, x12, x13, x14 as temporaries. 392.macro SETUP_STACK_FRAME code_item, refs, fp, cfi_refs, load_ins 393 FETCH_CODE_ITEM_INFO \code_item, wip, wip2, w15, \load_ins 394 395 // Compute required frame size: ((2 * ip) + ip2) * 4 + 24 396 // 24 is for saving the previous frame, pc, and method being executed. 397 add x14, ip, ip 398 add x14, x14, ip2 399 lsl x14, x14, #2 400 add x14, x14, #24 401 402 // Compute new stack pointer in x14 403 sub x14, sp, x14 404 // Alignment 405 and x14, x14, #-16 406 407 // Set reference and dex registers, align to pointer size for previous frame and dex pc. 408 add \refs, x14, ip2, lsl #2 409 add \refs, \refs, 28 410 and \refs, \refs, #(-__SIZEOF_POINTER__) 411 add \fp, \refs, ip, lsl #2 412 413 // Now setup the stack pointer. 414 mov x28, sp 415 .cfi_def_cfa_register x28 416 mov sp, x14 417 str x28, [\refs, #-8] 418 CFI_DEF_CFA_BREG_PLUS_UCONST \cfi_refs, -8, CALLEE_SAVES_SIZE 419 420 // Put nulls in reference frame. 421 cbz ip, 2f 422 mov ip2, \refs 4231: 424 str xzr, [ip2], #8 // May clear vreg[0]. 425 cmp ip2, \fp 426 b.lo 1b 4272: 428 // Save the ArtMethod. 429 str x0, [sp] 430.endm 431 432// Increase method hotness and do suspend check before starting executing the method. 433.macro START_EXECUTING_INSTRUCTIONS 434 ldr x0, [sp] 435 ldrh w2, [x0, #ART_METHOD_HOTNESS_COUNT_OFFSET] 436#if (NTERP_HOTNESS_VALUE != 0) 437#error Expected 0 for hotness value 438#endif 439 // If the counter is at zero, handle this in the runtime. 440 cbz w2, 3f 441 add x2, x2, #-1 442 strh w2, [x0, #ART_METHOD_HOTNESS_COUNT_OFFSET] 4431: 444 DO_SUSPEND_CHECK continue_label=2f 4452: 446 FETCH_INST 447 GET_INST_OPCODE ip 448 GOTO_OPCODE ip 4493: 450 CHECK_AND_UPDATE_SHARED_MEMORY_METHOD if_hot=4f, if_not_hot=1b 4514: 452 mov x1, xzr 453 mov x2, xFP 454 bl nterp_hot_method 455 b 2b 456.endm 457 458.macro SPILL_ALL_CALLEE_SAVES 459 INCREASE_FRAME CALLEE_SAVES_SIZE 460 // Note: we technically don't need to save x19 and x20, 461 // but the runtime will expect those values to be there when unwinding 462 // (see Arm64Context::DoLongJump checking for the thread register). 463 SAVE_ALL_CALLEE_SAVES 0 464.endm 465 466.macro RESTORE_ALL_CALLEE_SAVES 467 // FP callee-saves 468 ldp d8, d9, [sp, #0] 469 ldp d10, d11, [sp, #16] 470 ldp d12, d13, [sp, #32] 471 ldp d14, d15, [sp, #48] 472 473 // GP callee-saves. 474 // No need to restore x19 (it's always the thread), and 475 // don't restore x20 (the marking register) as it may have been updated, 476 // don't restore x21 (the suspend check register) as it may have been updated. 477 RESTORE_REG x22, 88 478 RESTORE_TWO_REGS x23, x24, 96 479 RESTORE_TWO_REGS x25, x26, 112 480 RESTORE_TWO_REGS x27, x28, 128 481 RESTORE_TWO_REGS x29, lr, 144 482 483 DECREASE_FRAME CALLEE_SAVES_SIZE 484.endm 485 486.macro SPILL_ALL_ARGUMENTS 487 stp x0, x1, [sp, #-128]! 488 stp x2, x3, [sp, #16] 489 stp x4, x5, [sp, #32] 490 stp x6, x7, [sp, #48] 491 stp d0, d1, [sp, #64] 492 stp d2, d3, [sp, #80] 493 stp d4, d5, [sp, #96] 494 stp d6, d7, [sp, #112] 495.endm 496 497.macro RESTORE_ALL_ARGUMENTS 498 ldp x2, x3, [sp, #16] 499 ldp x4, x5, [sp, #32] 500 ldp x6, x7, [sp, #48] 501 ldp d0, d1, [sp, #64] 502 ldp d2, d3, [sp, #80] 503 ldp d4, d5, [sp, #96] 504 ldp d6, d7, [sp, #112] 505 ldp x0, x1, [sp], #128 506.endm 507 508// Helper to setup the stack after doing a nterp to nterp call. This will setup: 509// - xNEW_FP: the new pointer to dex registers 510// - xNEW_REFS: the new pointer to references 511// - xPC: the new PC pointer to execute 512// - x2: value in instruction to decode the number of arguments. 513// - x3: first dex register 514// - x4: top of dex register array 515// 516// The method expects: 517// - x0 to contain the ArtMethod 518// - x8 to contain the code item 519.macro SETUP_STACK_FOR_INVOKE 520 // We do the same stack overflow check as the compiler. See CanMethodUseNterp 521 // in how we limit the maximum nterp frame size. 522 sub x16, sp, #STACK_OVERFLOW_RESERVED_BYTES 523 ldr wzr, [x16] 524 525 // Spill all callee saves to have a consistent stack frame whether we 526 // are called by compiled code or nterp. 527 SPILL_ALL_CALLEE_SAVES 528 529 // Setup the frame. 530 SETUP_STACK_FRAME x8, xNEW_REFS, xNEW_FP, CFI_NEW_REFS, load_ins=0 531 // Make x4 point to the top of the dex register array. 532 add x4, xNEW_FP, ip, uxtx #2 533 534 // Fetch instruction information before replacing xPC. 535 // TODO: move this down to the method that uses it, fetching it directly from wINST. 536 FETCH_B w2, 0, 1 537 // TODO: we could avoid this as instance invokes already fetch it. 538 FETCH w3, 2 539 540 // Set the dex pc pointer. 541 mov xPC, x8 542 CFI_DEFINE_DEX_PC_WITH_OFFSET(CFI_TMP, CFI_DEX, 0) 543.endm 544 545// Setup arguments based on a non-range nterp to nterp call, and start executing 546// the method. We expect: 547// - xNEW_FP: the new pointer to dex registers 548// - xNEW_REFS: the new pointer to references 549// - xPC: the new PC pointer to execute 550// - x2: number of arguments (bits 4-7), 5th argument if any (bits 0-3) 551// - x3: first dex register 552// - x4: top of dex register array 553// - x1: receiver if non-static. 554.macro SETUP_NON_RANGE_ARGUMENTS_AND_EXECUTE is_static=0, is_string_init=0 555 // /* op vA, vB, {vC...vG} */ 556 asr ip2, x2, #4 557 cbz ip2, 6f 558 mov ip, #-4 559 cmp ip2, #2 560 b.lt 1f 561 b.eq 2f 562 cmp ip2, #4 563 b.lt 3f 564 b.eq 4f 565 566 // We use a decrementing ip to store references relative 567 // to xNEW_FP and dex registers relative to x4 568 // 569 // TODO: We could set up ip as the number of registers (this can be an additional output from 570 // SETUP_STACK_FOR_INVOKE) and then just decrement it by one before copying each arg. 571 // Maybe even introduce macros NEW_VREG_ADDRESS/NEW_VREG_REF_ADDRESS. 5725: 573 and x2, x2, #15 574 GET_VREG_OBJECT w5, w2 575 str w5, [xNEW_FP, ip] 576 GET_VREG w5, w2 577 str w5, [x4, ip] 578 sub ip, ip, #4 5794: 580 asr x2, x3, #12 581 GET_VREG_OBJECT w5, w2 582 str w5, [xNEW_FP, ip] 583 GET_VREG w5, w2 584 str w5, [x4, ip] 585 sub ip, ip, #4 5863: 587 ubfx x2, x3, #8, #4 588 GET_VREG_OBJECT w5, w2 589 str w5, [xNEW_FP, ip] 590 GET_VREG w5, w2 591 str w5, [x4, ip] 592 sub ip, ip, #4 5932: 594 ubfx x2, x3, #4, #4 595 GET_VREG_OBJECT w5, w2 596 str w5, [xNEW_FP, ip] 597 GET_VREG w5, w2 598 str w5, [x4, ip] 599 .if !\is_string_init 600 sub ip, ip, #4 601 .endif 6021: 603 .if \is_string_init 604 // Ignore the first argument 605 .elseif \is_static 606 and x2, x3, #0xf 607 GET_VREG_OBJECT w5, w2 608 str w5, [xNEW_FP, ip] 609 GET_VREG w5, w2 610 str w5, [x4, ip] 611 .else 612 str w1, [xNEW_FP, ip] 613 str w1, [x4, ip] 614 .endif 615 6166: 617 // Start executing the method. 618 mov xFP, xNEW_FP 619 mov xREFS, xNEW_REFS 620 CFI_DEF_CFA_BREG_PLUS_UCONST CFI_REFS, -8, CALLEE_SAVES_SIZE 621 START_EXECUTING_INSTRUCTIONS 622.endm 623 624// Setup arguments based on a range nterp to nterp call, and start executing 625// the method. 626// - xNEW_FP: the new pointer to dex registers 627// - xNEW_REFS: the new pointer to references 628// - xPC: the new PC pointer to execute 629// - x2: number of arguments 630// - x3: first dex register 631// - x4: top of dex register array 632// - x1: receiver if non-static. 633// 634// Uses ip, ip2, x5, x6 as temporaries. 635.macro SETUP_RANGE_ARGUMENTS_AND_EXECUTE is_static=0, is_string_init=0 636 mov ip, #-4 637 .if \is_string_init 638 // Ignore the first argument 639 sub x2, x2, #1 640 add x3, x3, #1 641 .elseif !\is_static 642 sub x2, x2, #1 643 add x3, x3, #1 644 .endif 645 646 cbz x2, 2f 647 add ip2, xREFS, x3, lsl #2 // pointer to first argument in reference array 648 add ip2, ip2, x2, lsl #2 // pointer to last argument in reference array 649 add x5, xFP, x3, lsl #2 // pointer to first argument in register array 650 add x6, x5, x2, lsl #2 // pointer to last argument in register array 6511: 652 ldr w7, [ip2, #-4]! 653 str w7, [xNEW_FP, ip] 654 sub x2, x2, 1 655 ldr w7, [x6, #-4]! 656 str w7, [x4, ip] 657 sub ip, ip, 4 658 cbnz x2, 1b 6592: 660 .if \is_string_init 661 // Ignore first argument 662 .elseif !\is_static 663 str w1, [xNEW_FP, ip] 664 str w1, [x4, ip] 665 .endif 666 mov xFP, xNEW_FP 667 mov xREFS, xNEW_REFS 668 CFI_DEF_CFA_BREG_PLUS_UCONST CFI_REFS, -8, CALLEE_SAVES_SIZE 669 START_EXECUTING_INSTRUCTIONS 670.endm 671 672.macro GET_SHORTY dest, is_interface, is_polymorphic, is_custom 673 stp x0, x1, [sp, #-16]! 674 .if \is_polymorphic 675 ldr x0, [sp, #16] 676 mov x1, xPC 677 bl NterpGetShortyFromInvokePolymorphic 678 .elseif \is_custom 679 ldr x0, [sp, #16] 680 mov x1, xPC 681 bl NterpGetShortyFromInvokeCustom 682 .elseif \is_interface 683 ldr x0, [sp, #16] 684 FETCH w1, 1 685 bl NterpGetShortyFromMethodId 686 .else 687 bl NterpGetShorty 688 .endif 689 mov \dest, x0 690 ldp x0, x1, [sp], #16 691.endm 692 693.macro GET_SHORTY_SLOW_PATH dest, is_interface 694 // Save all registers that can hold arguments in the fast path. 695 stp x0, x1, [sp, #-32]! 696 str w2, [sp, #16] 697 str s0, [sp, #20] 698 .if \is_interface 699 ldr x0, [sp, #32] 700 FETCH w1, 1 701 bl NterpGetShortyFromMethodId 702 .else 703 bl NterpGetShorty 704 .endif 705 mov \dest, x0 706 ldr w2, [sp, #16] 707 ldr s0, [sp, #20] 708 ldp x0, x1, [sp], #32 709.endm 710 711// Input: x0 contains the ArtMethod 712// Output: x8 contains the code item 713.macro GET_CODE_ITEM 714 ldr x8, [x0, #ART_METHOD_DATA_OFFSET_64] 715.endm 716 717.macro DO_ENTRY_POINT_CHECK call_compiled_code 718 // On entry, the method is x0, the instance is x1 719 adr x2, ExecuteNterpImpl 720 ldr x3, [x0, #ART_METHOD_QUICK_CODE_OFFSET_64] 721 cmp x2, x3 722 b.ne \call_compiled_code 723.endm 724 725.macro UPDATE_REGISTERS_FOR_STRING_INIT old_value, new_value 726 mov wip, wzr 7271: 728 GET_VREG_OBJECT wip2, wip 729 cmp wip2, \old_value 730 b.ne 2f 731 SET_VREG_OBJECT \new_value, wip 7322: 733 add wip, wip, #1 734 add ip2, xREFS, wip, uxtw #2 735 cmp ip2, xFP 736 b.ne 1b 737.endm 738 739// Puts the next floating point argument into the expected register, 740// fetching values based on a non-range invoke. 741// Uses ip and ip2. 742.macro LOOP_OVER_SHORTY_LOADING_FPS dreg, sreg, inst, shorty, arg_index, finished 7431: // LOOP 744 ldrb wip, [\shorty], #1 // Load next character in shorty, and increment. 745 cbz wip, \finished // if (wip == '\0') goto finished 746 cmp wip, #68 // if (wip == 'D') goto FOUND_DOUBLE 747 b.eq 2f 748 cmp wip, #70 // if (wip == 'F') goto FOUND_FLOAT 749 b.eq 3f 750 lsr \inst, \inst, #4 751 add \arg_index, \arg_index, #1 752 // Handle extra argument in arg array taken by a long. 753 cmp wip, #74 // if (wip != 'J') goto LOOP 754 b.ne 1b 755 lsr \inst, \inst, #4 756 add \arg_index, \arg_index, #1 757 b 1b // goto LOOP 7582: // FOUND_DOUBLE 759 and ip, \inst, #0xf 760 GET_VREG wip, wip 761 lsr \inst, \inst, #4 762 add \arg_index, \arg_index, #1 763 cmp \arg_index, #4 764 b.eq 5f 765 and ip2, \inst, #0xf 766 lsr \inst, \inst, #4 767 add \arg_index, \arg_index, #1 768 b 6f 7695: 770 // TODO: Extract from wINST here and below? (Requires using a different register 771 // in the COMMON_INVOKE_NON_RANGE.) 772 FETCH_B wip2, 0, 1 773 and wip2, wip2, #0xf 7746: 775 GET_VREG wip2, wip2 776 add ip, ip, ip2, lsl #32 777 fmov \dreg, ip 778 b 4f 7793: // FOUND_FLOAT 780 cmp \arg_index, #4 781 b.eq 7f 782 and ip, \inst, #0xf 783 lsr \inst, \inst, #4 784 add \arg_index, \arg_index, #1 785 b 8f 7867: 787 FETCH_B wip, 0, 1 788 and wip, wip, #0xf 7898: 790 GET_VREG \sreg, wip 7914: 792.endm 793 794// Puts the next int/long/object argument in the expected register, 795// fetching values based on a non-range invoke. 796// Uses ip and ip2. 797.macro LOOP_OVER_SHORTY_LOADING_GPRS gpr_reg64, gpr_reg32, inst, shorty, arg_index, finished 7981: // LOOP 799 ldrb wip, [\shorty], #1 // Load next character in shorty, and increment. 800 cbz wip, \finished // if (wip == '\0') goto finished 801 cmp wip, #74 // if (wip == 'J') goto FOUND_LONG 802 b.eq 2f 803 cmp wip, #70 // if (wip == 'F') goto SKIP_FLOAT 804 b.eq 3f 805 cmp wip, #68 // if (wip == 'D') goto SKIP_DOUBLE 806 b.eq 4f 807 cmp \arg_index, #4 808 b.eq 7f 809 and ip, \inst, #0xf 810 lsr \inst, \inst, #4 811 add \arg_index, \arg_index, #1 812 b 8f 8137: 814 FETCH_B wip, 0, 1 815 and wip, wip, #0xf 8168: 817 GET_VREG \gpr_reg32, wip 818 b 5f 8192: // FOUND_LONG 820 and ip, \inst, #0xf 821 GET_VREG wip, wip 822 lsr \inst, \inst, #4 823 add \arg_index, \arg_index, #1 824 cmp \arg_index, #4 825 b.eq 9f 826 and ip2, \inst, #0xf 827 lsr \inst, \inst, #4 828 add \arg_index, \arg_index, #1 829 b 10f 8309: 831 FETCH_B wip2, 0, 1 832 and wip2, wip2, #0xf 83310: 834 GET_VREG wip2, wip2 835 add \gpr_reg64, ip, ip2, lsl #32 836 b 5f 8373: // SKIP_FLOAT 838 lsr \inst, \inst, #4 839 add \arg_index, \arg_index, #1 840 b 1b 8414: // SKIP_DOUBLE 842 lsr \inst, \inst, #4 843 add \arg_index, \arg_index, #1 844 cmp \arg_index, #4 845 b.eq 1b 846 lsr \inst, \inst, #4 847 add \arg_index, \arg_index, #1 848 b 1b 8495: 850.endm 851 852.macro SETUP_RETURN_VALUE shorty 853 ldrb wip, [\shorty] 854 cmp ip, #68 // Test if result type char == 'D'. 855 b.eq 1f 856 cmp ip, #70 // Test if result type char == 'F'. 857 b.ne 2f 858 fmov w0, s0 859 b 2f 8601: 861 fmov x0, d0 8622: 863.endm 864 865.macro COMMON_INVOKE_NON_RANGE is_static=0, is_interface=0, suffix="", is_string_init=0, is_polymorphic=0, is_custom=0 866 .if \is_polymorphic 867 // We always go to compiled code for polymorphic calls. 868 .elseif \is_custom 869 // We always go to compiled code for custom calls. 870 .else 871 DO_ENTRY_POINT_CHECK .Lcall_compiled_code_\suffix 872 GET_CODE_ITEM 873 .if \is_string_init 874 bl nterp_to_nterp_string_init_non_range 875 .elseif \is_static 876 bl nterp_to_nterp_static_non_range 877 .else 878 bl nterp_to_nterp_instance_non_range 879 .endif 880 b .Ldone_return_\suffix 881 .endif 882 883.Lcall_compiled_code_\suffix: 884 .if \is_polymorphic 885 // No fast path for polymorphic calls. 886 .elseif \is_custom 887 // No fast path for custom calls. 888 .elseif \is_string_init 889 // No fast path for string.init. 890 .else 891 ldr wip, [x0, #ART_METHOD_ACCESS_FLAGS_OFFSET] 892 tbz wip, #ART_METHOD_NTERP_INVOKE_FAST_PATH_FLAG_BIT, .Lfast_path_with_few_args_\suffix 893 FETCH_B wip2, 0, 1 894 asr ip, ip2, #4 895 .if \is_static 896 cbz ip, .Linvoke_fast_path_\suffix 897 .else 898 cmp ip, #1 899 b.eq .Linvoke_fast_path_\suffix 900 .endif 901 FETCH w8, 2 902 cmp ip, #2 903 .if \is_static 904 b.lt .Lone_arg_fast_path_\suffix 905 .endif 906 b.eq .Ltwo_args_fast_path_\suffix 907 cmp ip, #4 908 b.lt .Lthree_args_fast_path_\suffix 909 b.eq .Lfour_args_fast_path_\suffix 910 911 and ip, ip2, #15 912 GET_VREG w5, wip 913.Lfour_args_fast_path_\suffix: 914 asr ip, x8, #12 915 GET_VREG w4, wip 916.Lthree_args_fast_path_\suffix: 917 ubfx ip, x8, #8, #4 918 GET_VREG w3, wip 919.Ltwo_args_fast_path_\suffix: 920 ubfx ip, x8, #4, #4 921 GET_VREG w2, wip 922.Lone_arg_fast_path_\suffix: 923 .if \is_static 924 and ip, x8, #0xf 925 GET_VREG w1, wip 926 .else 927 // First argument already in w1. 928 .endif 929.Linvoke_fast_path_\suffix: 930 .if \is_interface 931 // Setup hidden argument. 932 mov ip2, x26 933 .endif 934 ldr lr, [x0, #ART_METHOD_QUICK_CODE_OFFSET_64] 935 blr lr 936 FETCH_ADVANCE_INST 3 937 GET_INST_OPCODE ip 938 GOTO_OPCODE ip 939 940.Lfast_path_with_few_args_\suffix: 941 // Fast path when we have zero or one argument (modulo 'this'). If there 942 // is one argument, we can put it in both floating point and core register. 943 FETCH_B w2, 0, 1 944 .if \is_static 945 cmp w2, #(2 << 4) 946 .else 947 cmp w2, #(3 << 4) 948 .endif 949 b.ge .Lget_shorty_\suffix 950 .if \is_static 951 tbz w2, #4, .Linvoke_with_few_args_\suffix 952 .else 953 tbnz w2, #4, .Linvoke_with_few_args_\suffix 954 .endif 955 FETCH w2, 2 956 .if \is_static 957 and w2, w2, #0xf // dex register of first argument 958 GET_VREG w1, w2 959 fmov s0, w1 960 .else 961 ubfx x2, x2, #4, #4 // dex register of second argument 962 GET_VREG w2, w2 963 fmov s0, w2 964 .endif 965.Linvoke_with_few_args_\suffix: 966 // Check if the next instruction is move-result or move-result-wide. 967 // If it is, we fetch the shorty and jump to the regular invocation. 968 FETCH w27, 3 969 and ip, x27, #0xfe 970 cmp ip, #0x0a 971 b.eq .Lget_shorty_and_invoke_\suffix 972 .if \is_interface 973 // Setup hidden argument. 974 mov ip2, x26 975 .endif 976 ldr lr, [x0, #ART_METHOD_QUICK_CODE_OFFSET_64] 977 blr lr 978 # TODO: Use some other register for shorty and prefetch the instruction directly to wINST. 979 mov xINST, x27 980 ADVANCE 3 981 GET_INST_OPCODE ip 982 GOTO_OPCODE ip 983.Lget_shorty_and_invoke_\suffix: 984 GET_SHORTY_SLOW_PATH xINST, \is_interface 985 b .Lgpr_setup_finished_\suffix 986 .endif 987 988.Lget_shorty_\suffix: 989 GET_SHORTY xINST, \is_interface, \is_polymorphic, \is_custom 990 // From this point: 991 // - xINST contains shorty (in callee-save to switch over return value after call). 992 // - x0 contains method 993 // - x1 contains 'this' pointer for instance method. 994 // - for interface calls, x26 contains the interface method. 995 add x9, xINST, #1 // shorty + 1 ; ie skip return arg character 996 FETCH w11, 2 // arguments 997 .if \is_string_init 998 lsr x11, x11, #4 999 mov x10, #1 // ignore first argument 1000 .elseif \is_static 1001 mov x10, xzr // arg_index 1002 .else 1003 lsr x11, x11, #4 1004 mov x10, #1 // ignore first argument 1005 .endif 1006 LOOP_OVER_SHORTY_LOADING_FPS d0, s0, x11, x9, x10, .Lxmm_setup_finished_\suffix 1007 LOOP_OVER_SHORTY_LOADING_FPS d1, s1, x11, x9, x10, .Lxmm_setup_finished_\suffix 1008 LOOP_OVER_SHORTY_LOADING_FPS d2, s2, x11, x9, x10, .Lxmm_setup_finished_\suffix 1009 LOOP_OVER_SHORTY_LOADING_FPS d3, s3, x11, x9, x10, .Lxmm_setup_finished_\suffix 1010 LOOP_OVER_SHORTY_LOADING_FPS d4, s4, x11, x9, x10, .Lxmm_setup_finished_\suffix 1011.Lxmm_setup_finished_\suffix: 1012 add x9, xINST, #1 // shorty + 1 ; ie skip return arg character 1013 FETCH w11, 2 // arguments 1014 .if \is_string_init 1015 lsr x11, x11, #4 1016 mov x10, #1 // ignore first argument 1017 LOOP_OVER_SHORTY_LOADING_GPRS x1, w1, x11, x9, x10, .Lgpr_setup_finished_\suffix 1018 .elseif \is_static 1019 mov x10, xzr // arg_index 1020 LOOP_OVER_SHORTY_LOADING_GPRS x1, w1, x11, x9, x10, .Lgpr_setup_finished_\suffix 1021 .else 1022 lsr x11, x11, #4 1023 mov x10, #1 // ignore first argument 1024 .endif 1025 LOOP_OVER_SHORTY_LOADING_GPRS x2, w2, x11, x9, x10, .Lgpr_setup_finished_\suffix 1026 LOOP_OVER_SHORTY_LOADING_GPRS x3, w3, x11, x9, x10, .Lgpr_setup_finished_\suffix 1027 LOOP_OVER_SHORTY_LOADING_GPRS x4, w4, x11, x9, x10, .Lgpr_setup_finished_\suffix 1028 LOOP_OVER_SHORTY_LOADING_GPRS x5, w5, x11, x9, x10, .Lgpr_setup_finished_\suffix 1029.Lgpr_setup_finished_\suffix: 1030 .if \is_polymorphic 1031 bl art_quick_invoke_polymorphic 1032 .elseif \is_custom 1033 bl art_quick_invoke_custom 1034 .else 1035 .if \is_interface 1036 // Setup hidden argument. 1037 mov ip2, x26 1038 .endif 1039 ldr lr, [x0, #ART_METHOD_QUICK_CODE_OFFSET_64] 1040 blr lr 1041 .endif 1042 SETUP_RETURN_VALUE xINST 1043.Ldone_return_\suffix: 1044 /* resume execution of caller */ 1045 .if \is_string_init 1046 FETCH w11, 2 // arguments 1047 and x11, x11, #0xf 1048 GET_VREG w1, w11 1049 UPDATE_REGISTERS_FOR_STRING_INIT w1, w0 1050 .endif 1051 1052 .if \is_polymorphic 1053 FETCH_ADVANCE_INST 4 1054 .else 1055 FETCH_ADVANCE_INST 3 1056 .endif 1057 GET_INST_OPCODE ip 1058 GOTO_OPCODE ip 1059.endm 1060 1061// Puts the next floating point argument into the expected register, 1062// fetching values based on a range invoke. 1063// Uses ip as temporary. 1064.macro LOOP_RANGE_OVER_SHORTY_LOADING_FPS dreg, sreg, shorty, arg_index, stack_index, finished 10651: // LOOP 1066 ldrb wip, [\shorty], #1 // Load next character in shorty, and increment. 1067 cbz wip, \finished // if (wip == '\0') goto finished 1068 cmp wip, #68 // if (wip == 'D') goto FOUND_DOUBLE 1069 b.eq 2f 1070 cmp wip, #70 // if (wip == 'F') goto FOUND_FLOAT 1071 b.eq 3f 1072 add \arg_index, \arg_index, #1 1073 add \stack_index, \stack_index, #1 1074 // Handle extra argument in arg array taken by a long. 1075 cmp wip, #74 // if (wip != 'J') goto LOOP 1076 b.ne 1b 1077 add \arg_index, \arg_index, #1 1078 add \stack_index, \stack_index, #1 1079 b 1b // goto LOOP 10802: // FOUND_DOUBLE 1081 GET_VREG_DOUBLE \dreg, \arg_index 1082 add \arg_index, \arg_index, #2 1083 add \stack_index, \stack_index, #2 1084 b 4f 10853: // FOUND_FLOAT 1086 GET_VREG \sreg, \arg_index 1087 add \arg_index, \arg_index, #1 1088 add \stack_index, \stack_index, #1 10894: 1090.endm 1091 1092// Puts the next floating point argument into the expected stack slot, 1093// fetching values based on a range invoke. 1094// Uses ip as temporary. 1095// 1096// TODO: We could just copy all the vregs to the stack slots in a simple loop 1097// without looking at the shorty at all. (We could also drop 1098// the "stack_index" from the macros for loading registers.) We could also do 1099// that conditionally if argument word count > 6; otherwise we know that all 1100// args fit into registers. 1101.macro LOOP_RANGE_OVER_FPs shorty, arg_index, stack_index, finished 11021: // LOOP 1103 ldrb wip, [\shorty], #1 // Load next character in shorty, and increment. 1104 cbz wip, \finished // if (wip == '\0') goto finished 1105 cmp wip, #68 // if (wip == 'D') goto FOUND_DOUBLE 1106 b.eq 2f 1107 cmp wip, #70 // if (wip == 'F') goto FOUND_FLOAT 1108 b.eq 3f 1109 add \arg_index, \arg_index, #1 1110 add \stack_index, \stack_index, #1 1111 // Handle extra argument in arg array taken by a long. 1112 cmp wip, #74 // if (wip != 'J') goto LOOP 1113 b.ne 1b 1114 add \arg_index, \arg_index, #1 1115 add \stack_index, \stack_index, #1 1116 b 1b // goto LOOP 11172: // FOUND_DOUBLE 1118 GET_VREG_WIDE ip, \arg_index 1119 add ip2, sp, \stack_index, uxtw #2 1120 str ip, [ip2] 1121 add \arg_index, \arg_index, #2 1122 add \stack_index, \stack_index, #2 1123 b 1b 11243: // FOUND_FLOAT 1125 GET_VREG wip, \arg_index 1126 str wip, [sp, \stack_index, uxtw #2] 1127 add \arg_index, \arg_index, #1 1128 add \stack_index, \stack_index, #1 1129 b 1b 1130.endm 1131 1132// Puts the next int/long/object argument in the expected register, 1133// fetching values based on a range invoke. 1134// Uses ip as temporary. 1135.macro LOOP_RANGE_OVER_SHORTY_LOADING_GPRS reg64, reg32, shorty, arg_index, stack_index, finished 11361: // LOOP 1137 ldrb wip, [\shorty], #1 // Load next character in shorty, and increment. 1138 cbz wip, \finished // if (wip == '\0') goto finished 1139 cmp wip, #74 // if (wip == 'J') goto FOUND_LONG 1140 b.eq 2f 1141 cmp wip, #70 // if (wip == 'F') goto SKIP_FLOAT 1142 b.eq 3f 1143 cmp wip, #68 // if (wip == 'D') goto SKIP_DOUBLE 1144 b.eq 4f 1145 GET_VREG \reg32, \arg_index 1146 add \arg_index, \arg_index, #1 1147 add \stack_index, \stack_index, #1 1148 b 5f 11492: // FOUND_LONG 1150 GET_VREG_WIDE \reg64, \arg_index 1151 add \arg_index, \arg_index, #2 1152 add \stack_index, \stack_index, #2 1153 b 5f 11543: // SKIP_FLOAT 1155 add \arg_index, \arg_index, #1 1156 add \stack_index, \stack_index, #1 1157 b 1b 11584: // SKIP_DOUBLE 1159 add \arg_index, \arg_index, #2 1160 add \stack_index, \stack_index, #2 1161 b 1b 11625: 1163.endm 1164 1165// Puts the next int/long/object argument in the expected stack slot, 1166// fetching values based on a range invoke. 1167// Uses ip as temporary. 1168.macro LOOP_RANGE_OVER_INTs shorty, arg_index, stack_index, finished 11691: // LOOP 1170 ldrb wip, [\shorty], #1 // Load next character in shorty, and increment. 1171 cbz wip, \finished // if (wip == '\0') goto finished 1172 cmp wip, #74 // if (wip == 'J') goto FOUND_LONG 1173 b.eq 2f 1174 cmp wip, #70 // if (wip == 'F') goto SKIP_FLOAT 1175 b.eq 3f 1176 cmp wip, #68 // if (wip == 'D') goto SKIP_DOUBLE 1177 b.eq 4f 1178 GET_VREG wip, \arg_index 1179 str wip, [sp, \stack_index, uxtw #2] 1180 add \arg_index, \arg_index, #1 1181 add \stack_index, \stack_index, #1 1182 b 1b 11832: // FOUND_LONG 1184 GET_VREG_WIDE ip, \arg_index 1185 add ip2, sp, \stack_index, uxtw #2 1186 str ip, [ip2] 1187 add \arg_index, \arg_index, #2 1188 add \stack_index, \stack_index, #2 1189 b 1b 11903: // SKIP_FLOAT 1191 add \arg_index, \arg_index, #1 1192 add \stack_index, \stack_index, #1 1193 b 1b 11944: // SKIP_DOUBLE 1195 add \arg_index, \arg_index, #2 1196 add \stack_index, \stack_index, #2 1197 b 1b 1198.endm 1199 1200.macro COMMON_INVOKE_RANGE is_static=0, is_interface=0, suffix="", is_string_init=0, is_polymorphic=0, is_custom=0 1201 .if \is_polymorphic 1202 // We always go to compiled code for polymorphic calls. 1203 .elseif \is_custom 1204 // We always go to compiled code for custom calls. 1205 .else 1206 DO_ENTRY_POINT_CHECK .Lcall_compiled_code_range_\suffix 1207 GET_CODE_ITEM 1208 .if \is_string_init 1209 bl nterp_to_nterp_string_init_range 1210 .elseif \is_static 1211 bl nterp_to_nterp_static_range 1212 .else 1213 bl nterp_to_nterp_instance_range 1214 .endif 1215 b .Ldone_return_range_\suffix 1216 .endif 1217 1218.Lcall_compiled_code_range_\suffix: 1219 .if \is_polymorphic 1220 // No fast path for polymorphic calls. 1221 .elseif \is_custom 1222 // No fast path for custom calls. 1223 .elseif \is_string_init 1224 // No fast path for string.init. 1225 .else 1226 ldr wip, [x0, #ART_METHOD_ACCESS_FLAGS_OFFSET] 1227 tbz wip, #ART_METHOD_NTERP_INVOKE_FAST_PATH_FLAG_BIT, .Lfast_path_with_few_args_range_\suffix 1228 FETCH_B wip2, 0, 1 // Number of arguments 1229 .if \is_static 1230 cbz ip2, .Linvoke_fast_path_range_\suffix 1231 .else 1232 cmp ip2, #1 1233 b.eq .Linvoke_fast_path_range_\suffix 1234 .endif 1235 FETCH wip, 2 // dex register of first argument 1236 add x8, xFP, wip, uxtw #2 // location of first dex register value 1237 cmp ip2, #2 1238 .if \is_static 1239 b.lt .Lone_arg_fast_path_range_\suffix 1240 .endif 1241 b.eq .Ltwo_args_fast_path_range_\suffix 1242 cmp ip2, #4 1243 b.lt .Lthree_args_fast_path_range_\suffix 1244 b.eq .Lfour_args_fast_path_range_\suffix 1245 cmp ip2, #6 1246 b.lt .Lfive_args_fast_path_range_\suffix 1247 b.eq .Lsix_args_fast_path_range_\suffix 1248 cmp ip2, #7 1249 b.eq .Lseven_args_fast_path_range_\suffix 1250 // Setup x8 to point to the stack location of parameters we do not need 1251 // to put parameters in. 1252 add x9, sp, #8 // Add space for the ArtMethod 1253 1254.Lloop_over_fast_path_range_\suffix: 1255 sub ip2, ip2, #1 1256 ldr wip, [x8, ip2, lsl #2] 1257 str wip, [x9, ip2, lsl #2] 1258 cmp ip2, #7 1259 b.ne .Lloop_over_fast_path_range_\suffix 1260 1261.Lseven_args_fast_path_range_\suffix: 1262 ldr w7, [x8, #24] 1263.Lsix_args_fast_path_range_\suffix: 1264 ldr w6, [x8, #20] 1265.Lfive_args_fast_path_range_\suffix: 1266 ldr w5, [x8, #16] 1267.Lfour_args_fast_path_range_\suffix: 1268 ldr w4, [x8, #12] 1269.Lthree_args_fast_path_range_\suffix: 1270 ldr w3, [x8, #8] 1271.Ltwo_args_fast_path_range_\suffix: 1272 ldr w2, [x8, #4] 1273.Lone_arg_fast_path_range_\suffix: 1274 .if \is_static 1275 ldr w1, [x8, #0] 1276 .else 1277 // First argument already in w1. 1278 .endif 1279.Linvoke_fast_path_range_\suffix: 1280 .if \is_interface 1281 // Setup hidden argument. 1282 mov ip2, x26 1283 .endif 1284 ldr lr, [x0, #ART_METHOD_QUICK_CODE_OFFSET_64] 1285 blr lr 1286 FETCH_ADVANCE_INST 3 1287 GET_INST_OPCODE ip 1288 GOTO_OPCODE ip 1289 1290.Lfast_path_with_few_args_range_\suffix: 1291 // Fast path when we have zero or one argument (modulo 'this'). If there 1292 // is one argument, we can put it in both floating point and core register. 1293 FETCH_B w2, 0, 1 // number of arguments 1294 .if \is_static 1295 cmp w2, #1 1296 .else 1297 cmp w2, #2 1298 .endif 1299 b.lt .Linvoke_with_few_args_range_\suffix 1300 b.ne .Lget_shorty_range_\suffix 1301 FETCH w3, 2 // dex register of first argument 1302 .if \is_static 1303 GET_VREG w1, w3 1304 fmov s0, w1 1305 .else 1306 add w3, w3, #1 // Add 1 for next argument 1307 GET_VREG w2, w3 1308 fmov s0, w2 1309 .endif 1310.Linvoke_with_few_args_range_\suffix: 1311 // Check if the next instruction is move-result or move-result-wide. 1312 // If it is, we fetch the shorty and jump to the regular invocation. 1313 FETCH w27, 3 1314 and ip, x27, #0xfe 1315 cmp ip, #0x0a 1316 b.eq .Lget_shorty_and_invoke_range_\suffix 1317 .if \is_interface 1318 // Setup hidden argument. 1319 mov ip2, x26 1320 .endif 1321 ldr lr, [x0, #ART_METHOD_QUICK_CODE_OFFSET_64] 1322 blr lr 1323 mov xINST, x27 1324 ADVANCE 3 1325 GET_INST_OPCODE ip 1326 GOTO_OPCODE ip 1327.Lget_shorty_and_invoke_range_\suffix: 1328 GET_SHORTY_SLOW_PATH xINST, \is_interface 1329 b .Lgpr_setup_finished_range_\suffix 1330 .endif 1331 1332.Lget_shorty_range_\suffix: 1333 GET_SHORTY xINST, \is_interface, \is_polymorphic, \is_custom 1334 // From this point: 1335 // - xINST contains shorty (in callee-save to switch over return value after call). 1336 // - x0 contains method 1337 // - x1 contains 'this' pointer for instance method. 1338 // - for interface calls, x26 contains the interface method. 1339 add x9, xINST, #1 // shorty + 1 ; ie skip return arg character 1340 FETCH w10, 2 // arguments 1341 .if \is_string_init 1342 add x10, x10, #1 // arg start index 1343 mov x11, #1 // index in stack 1344 .elseif \is_static 1345 mov x11, xzr // index in stack 1346 .else 1347 add x10, x10, #1 // arg start index 1348 mov x11, #1 // index in stack 1349 .endif 1350 LOOP_RANGE_OVER_SHORTY_LOADING_FPS d0, s0, x9, w10, w11, .Lxmm_setup_finished_range_\suffix 1351 LOOP_RANGE_OVER_SHORTY_LOADING_FPS d1, s1, x9, w10, w11, .Lxmm_setup_finished_range_\suffix 1352 LOOP_RANGE_OVER_SHORTY_LOADING_FPS d2, s2, x9, w10, w11, .Lxmm_setup_finished_range_\suffix 1353 LOOP_RANGE_OVER_SHORTY_LOADING_FPS d3, s3, x9, w10, w11, .Lxmm_setup_finished_range_\suffix 1354 LOOP_RANGE_OVER_SHORTY_LOADING_FPS d4, s4, x9, w10, w11, .Lxmm_setup_finished_range_\suffix 1355 LOOP_RANGE_OVER_SHORTY_LOADING_FPS d5, s5, x9, w10, w11, .Lxmm_setup_finished_range_\suffix 1356 LOOP_RANGE_OVER_SHORTY_LOADING_FPS d6, s6, x9, w10, w11, .Lxmm_setup_finished_range_\suffix 1357 LOOP_RANGE_OVER_SHORTY_LOADING_FPS d7, s7, x9, w10, w11, .Lxmm_setup_finished_range_\suffix 1358 // Store in the outs array (stored above the ArtMethod in the stack) 1359 add x11, x11, #2 // Add two words for the ArtMethod stored before the outs. 1360 LOOP_RANGE_OVER_FPs x9, w10, w11, .Lxmm_setup_finished_range_\suffix 1361.Lxmm_setup_finished_range_\suffix: 1362 add x9, xINST, #1 // shorty + 1 ; ie skip return arg character 1363 FETCH w10, 2 // arguments 1364 .if \is_string_init 1365 add x10, x10, #1 // arg start index 1366 mov x11, #1 // index in stack 1367 LOOP_RANGE_OVER_SHORTY_LOADING_GPRS x1, w1, x9, w10, w11, .Lgpr_setup_finished_range_\suffix 1368 .elseif \is_static 1369 mov x11, xzr // index in stack 1370 LOOP_RANGE_OVER_SHORTY_LOADING_GPRS x1, w1, x9, w10, w11 .Lgpr_setup_finished_range_\suffix 1371 .else 1372 add x10, x10, #1 // arg start index 1373 mov x11, #1 // index in stack 1374 .endif 1375 LOOP_RANGE_OVER_SHORTY_LOADING_GPRS x2, w2, x9, w10, w11, .Lgpr_setup_finished_range_\suffix 1376 LOOP_RANGE_OVER_SHORTY_LOADING_GPRS x3, w3, x9, w10, w11, .Lgpr_setup_finished_range_\suffix 1377 LOOP_RANGE_OVER_SHORTY_LOADING_GPRS x4, w4, x9, w10, w11, .Lgpr_setup_finished_range_\suffix 1378 LOOP_RANGE_OVER_SHORTY_LOADING_GPRS x5, w5, x9, w10, w11, .Lgpr_setup_finished_range_\suffix 1379 LOOP_RANGE_OVER_SHORTY_LOADING_GPRS x6, w6, x9, w10, w11, .Lgpr_setup_finished_range_\suffix 1380 LOOP_RANGE_OVER_SHORTY_LOADING_GPRS x7, w7, x9, w10, w11, .Lgpr_setup_finished_range_\suffix 1381 // Store in the outs array (stored above the ArtMethod in the stack) 1382 add x11, x11, #2 // Add two words for the ArtMethod stored before the outs. 1383 LOOP_RANGE_OVER_INTs x9, w10, w11, .Lgpr_setup_finished_range_\suffix 1384.Lgpr_setup_finished_range_\suffix: 1385 .if \is_polymorphic 1386 bl art_quick_invoke_polymorphic 1387 .elseif \is_custom 1388 bl art_quick_invoke_custom 1389 .else 1390 .if \is_interface 1391 // Setup hidden argument. 1392 mov ip2, x26 1393 .endif 1394 ldr lr, [x0, #ART_METHOD_QUICK_CODE_OFFSET_64] 1395 blr lr 1396 .endif 1397 SETUP_RETURN_VALUE xINST 1398.Ldone_return_range_\suffix: 1399 /* resume execution of caller */ 1400 .if \is_string_init 1401 FETCH w11, 2 // arguments 1402 GET_VREG w1, w11 1403 UPDATE_REGISTERS_FOR_STRING_INIT w1, w0 1404 .endif 1405 1406 .if \is_polymorphic 1407 FETCH_ADVANCE_INST 4 1408 .else 1409 FETCH_ADVANCE_INST 3 1410 .endif 1411 GET_INST_OPCODE ip 1412 GOTO_OPCODE ip 1413.endm 1414 1415.macro POISON_HEAP_REF_IF_OBJECT is_object, rRef 1416 .if \is_object 1417 POISON_HEAP_REF \rRef 1418 .endif 1419.endm 1420 1421.macro WRITE_BARRIER_IF_OBJECT is_object, value, holder, label 1422 .if \is_object 1423 cbz \value, \label 1424 ldr ip, [xSELF, #THREAD_CARD_TABLE_OFFSET] 1425 lsr wip2, \holder, #CARD_TABLE_CARD_SHIFT 1426 strb wip, [ip, ip2] 1427\label: 1428 .endif 1429.endm 1430 1431// Puts the next int/long/object parameter passed in physical register 1432// in the expected dex register array entry, and in case of object in the 1433// expected reference array entry. 1434.macro LOOP_OVER_SHORTY_STORING_GPRS gpr_64, gpr_32, shorty, arg_offset, regs, refs, finished 14351: // LOOP 1436 ldrb wip, [\shorty], #1 // Load next character in shorty, and increment. 1437 cbz wip, \finished // if (wip == '\0') goto finished 1438 cmp wip, #74 // if (wip == 'J') goto FOUND_LONG 1439 b.eq 2f 1440 cmp wip, #70 // if (wip == 'F') goto SKIP_FLOAT 1441 b.eq 3f 1442 cmp wip, #68 // if (wip == 'D') goto SKIP_DOUBLE 1443 b.eq 4f 1444 str \gpr_32, [\regs, \arg_offset] 1445 cmp wip, #76 // if (wip != 'L') goto NOT_REFERENCE 1446 b.ne 6f 1447 str \gpr_32, [\refs, \arg_offset] 14486: // NOT_REFERENCE 1449 add \arg_offset, \arg_offset, #4 1450 b 5f 14512: // FOUND_LONG 1452 str \gpr_64, [\regs, \arg_offset] 1453 add \arg_offset, \arg_offset, #8 1454 b 5f 14553: // SKIP_FLOAT 1456 add \arg_offset, \arg_offset, #4 1457 b 1b 14584: // SKIP_DOUBLE 1459 add \arg_offset, \arg_offset, #8 1460 b 1b 14615: 1462.endm 1463 1464// Puts the next floating point parameter passed in physical register 1465// in the expected dex register array entry. 1466// Uses ip as temporary. 1467.macro LOOP_OVER_SHORTY_STORING_FPS dreg, sreg, shorty, arg_offset, fp, finished 14681: // LOOP 1469 ldrb wip, [\shorty], #1 // Load next character in shorty, and increment. 1470 cbz wip, \finished // if (wip == '\0') goto finished 1471 cmp wip, #68 // if (wip == 'D') goto FOUND_DOUBLE 1472 b.eq 2f 1473 cmp wip, #70 // if (wip == 'F') goto FOUND_FLOAT 1474 b.eq 3f 1475 add \arg_offset, \arg_offset, #4 1476 // Handle extra argument in arg array taken by a long. 1477 cmp wip, #74 // if (wip != 'J') goto LOOP 1478 b.ne 1b 1479 add \arg_offset, \arg_offset, #4 1480 b 1b // goto LOOP 14812: // FOUND_DOUBLE 1482 str \dreg, [\fp, \arg_offset] 1483 add \arg_offset, \arg_offset, #8 1484 b 4f 14853: // FOUND_FLOAT 1486 str \sreg, [\fp, \arg_offset] 1487 add \arg_offset, \arg_offset, #4 14884: 1489.endm 1490 1491// Puts the next floating point parameter passed in stack 1492// in the expected dex register array entry. 1493// Uses ip as temporary. 1494// 1495// TODO: Or we could just spill regs to the reserved slots in the caller's 1496// frame and copy all regs in a simple loop. This time, however, we would 1497// need to look at the shorty anyway to look for the references. 1498// (The trade-off is different for passing arguments and receiving them.) 1499.macro LOOP_OVER_FPs shorty, arg_offset, regs, stack_ptr, finished 15001: // LOOP 1501 ldrb wip, [\shorty], #1 // Load next character in shorty, and increment. 1502 cbz wip, \finished // if (wip == '\0') goto finished 1503 cmp wip, #68 // if (wip == 'D') goto FOUND_DOUBLE 1504 b.eq 2f 1505 cmp wip, #70 // if (wip == 'F') goto FOUND_FLOAT 1506 b.eq 3f 1507 add \arg_offset, \arg_offset, #4 1508 // Handle extra argument in arg array taken by a long. 1509 cmp wip, #74 // if (wip != 'J') goto LOOP 1510 b.ne 1b 1511 add \arg_offset, \arg_offset, #4 1512 b 1b // goto LOOP 15132: // FOUND_DOUBLE 1514 add ip, \stack_ptr, \arg_offset 1515 ldr ip, [ip, #OFFSET_TO_FIRST_ARGUMENT_IN_STACK] 1516 str ip, [\regs, \arg_offset] 1517 add \arg_offset, \arg_offset, #8 1518 b 1b 15193: // FOUND_FLOAT 1520 add ip, \stack_ptr, \arg_offset 1521 ldr wip, [ip, #OFFSET_TO_FIRST_ARGUMENT_IN_STACK] 1522 str wip, [\regs, \arg_offset] 1523 add \arg_offset, \arg_offset, #4 1524 b 1b 1525.endm 1526 1527// Puts the next int/long/object parameter passed in stack 1528// in the expected dex register array entry, and in case of object in the 1529// expected reference array entry. 1530// Uses ip and ip2 as temporary. 1531.macro LOOP_OVER_INTs shorty, arg_offset, regs, refs, stack_ptr, finished 15321: // LOOP 1533 ldrb wip, [\shorty], #1 // Load next character in shorty, and increment. 1534 cbz wip, \finished // if (wip == '\0') goto finished 1535 cmp wip, #74 // if (wip == 'J') goto FOUND_LONG 1536 b.eq 2f 1537 cmp wip, #70 // if (wip == 'F') goto SKIP_FLOAT 1538 b.eq 3f 1539 cmp wip, #68 // if (wip == 'D') goto SKIP_DOUBLE 1540 b.eq 4f 1541 add ip2, \stack_ptr, \arg_offset 1542 ldr wip2, [ip2, #OFFSET_TO_FIRST_ARGUMENT_IN_STACK] 1543 str wip2, [\regs, \arg_offset] 1544 cmp wip, #76 // if (wip != 'L') goto loop 1545 b.ne 3f 1546 str wip2, [\refs, \arg_offset] 1547 add \arg_offset, \arg_offset, #4 1548 b 1b 15492: // FOUND_LONG 1550 add ip, \stack_ptr, \arg_offset 1551 ldr ip, [ip, #OFFSET_TO_FIRST_ARGUMENT_IN_STACK] 1552 str ip, [\regs, \arg_offset] 1553 add \arg_offset, \arg_offset, #8 1554 b 1b 15553: // SKIP_FLOAT 1556 add \arg_offset, \arg_offset, #4 1557 b 1b 15584: // SKIP_DOUBLE 1559 add \arg_offset, \arg_offset, #8 1560 b 1b 1561.endm 1562 1563.macro SETUP_REFERENCE_PARAMETER_IN_GPR gpr32, regs, refs, ins, arg_offset, finished 1564 str \gpr32, [\regs, \arg_offset] 1565 sub \ins, \ins, #1 1566 str \gpr32, [\refs, \arg_offset] 1567 add \arg_offset, \arg_offset, #4 1568 cbz \ins, \finished 1569.endm 1570 1571// Uses ip2 as temporary. 1572.macro SETUP_REFERENCE_PARAMETERS_IN_STACK regs, refs, ins, stack_ptr, arg_offset 15731: 1574 ldr wip2, [\stack_ptr, \arg_offset] 1575 sub \ins, \ins, #1 1576 str wip2, [\regs, \arg_offset] 1577 str wip2, [\refs, \arg_offset] 1578 add \arg_offset, \arg_offset, #4 1579 cbnz \ins, 1b 1580.endm 1581 1582.macro CHECK_AND_UPDATE_SHARED_MEMORY_METHOD if_hot, if_not_hot 1583 ldr wip, [x0, #ART_METHOD_ACCESS_FLAGS_OFFSET] 1584 tbz wip, #ART_METHOD_IS_MEMORY_SHARED_FLAG_BIT, \if_hot 1585 ldr wip, [xSELF, #THREAD_SHARED_METHOD_HOTNESS_OFFSET] 1586 cbz wip, \if_hot 1587 add wip, wip, #-1 1588 str wip, [xSELF, #THREAD_SHARED_METHOD_HOTNESS_OFFSET] 1589 b \if_not_hot 1590.endm 1591 1592.macro DO_SUSPEND_CHECK continue_label 1593 ldr wip, [xSELF, #THREAD_FLAGS_OFFSET] 1594 tst wip, #THREAD_SUSPEND_OR_CHECKPOINT_REQUEST 1595 b.eq \continue_label 1596 EXPORT_PC 1597 bl art_quick_test_suspend 1598.endm 1599 1600%def entry(): 1601/* 1602 * ArtMethod entry point. 1603 * 1604 * On entry: 1605 * x0 ArtMethod* callee 1606 * rest method parameters 1607 */ 1608 1609OAT_ENTRY ExecuteNterpWithClinitImpl 1610 .cfi_startproc 1611 // For simplicity, we don't do a read barrier here, but instead rely 1612 // on art_quick_resolution_trampoline to always have a suspend point before 1613 // calling back here. 1614 ldr wip, [x0, #ART_METHOD_DECLARING_CLASS_OFFSET] 1615 ldr wip2, [ip, #MIRROR_CLASS_STATUS_OFFSET] 1616 lsr wip2, wip2, #MIRROR_CLASS_STATUS_SHIFT 1617 cmp wip2, #MIRROR_CLASS_STATUS_VISIBLY_INITIALIZED 1618 b.hs ExecuteNterpImpl 1619 cmp wip2, #MIRROR_CLASS_STATUS_INITIALIZED 1620 b.lo .Linitializing_check 1621 dmb ish 1622 b ExecuteNterpImpl 1623.Linitializing_check: 1624 cmp wip2, #MIRROR_CLASS_STATUS_INITIALIZING 1625 b.lo .Lresolution_trampoline 1626 ldr wip2, [ip, #MIRROR_CLASS_CLINIT_THREAD_ID_OFFSET] 1627 ldr wip, [xSELF, #THREAD_TID_OFFSET] 1628 cmp wip, wip2 1629 b.eq ExecuteNterpImpl 1630.Lresolution_trampoline: 1631 b art_quick_resolution_trampoline 1632 .cfi_endproc 1633 .type EndExecuteNterpWithClinitImpl, #function 1634 .hidden EndExecuteNterpWithClinitImpl 1635 .global EndExecuteNterpWithClinitImpl 1636EndExecuteNterpWithClinitImpl: 1637 1638OAT_ENTRY ExecuteNterpImpl 1639 .cfi_startproc 1640 sub x16, sp, #STACK_OVERFLOW_RESERVED_BYTES 1641 ldr wzr, [x16] 1642 /* Spill callee save regs */ 1643 SPILL_ALL_CALLEE_SAVES 1644 1645 ldr xPC, [x0, #ART_METHOD_DATA_OFFSET_64] 1646 // Setup the stack for executing the method. 1647 SETUP_STACK_FRAME xPC, xREFS, xFP, CFI_REFS, load_ins=1 1648 1649 // Setup the parameters 1650 cbz w15, .Lxmm_setup_finished 1651 1652 sub ip2, ip, x15 1653 ldr w26, [x0, #ART_METHOD_ACCESS_FLAGS_OFFSET] 1654 lsl x27, ip2, #2 // x27 is now the offset for inputs into the registers array. 1655 1656 tbz w26, #ART_METHOD_NTERP_ENTRY_POINT_FAST_PATH_FLAG_BIT, .Lsetup_slow_path 1657 // Setup pointer to inputs in FP and pointer to inputs in REFS 1658 add x10, xFP, x27 1659 add x11, xREFS, x27 1660 mov x12, #0 1661 SETUP_REFERENCE_PARAMETER_IN_GPR w1, x10, x11, w15, x12, .Lxmm_setup_finished 1662 SETUP_REFERENCE_PARAMETER_IN_GPR w2, x10, x11, w15, x12, .Lxmm_setup_finished 1663 SETUP_REFERENCE_PARAMETER_IN_GPR w3, x10, x11, w15, x12, .Lxmm_setup_finished 1664 SETUP_REFERENCE_PARAMETER_IN_GPR w4, x10, x11, w15, x12, .Lxmm_setup_finished 1665 SETUP_REFERENCE_PARAMETER_IN_GPR w5, x10, x11, w15, x12, .Lxmm_setup_finished 1666 SETUP_REFERENCE_PARAMETER_IN_GPR w6, x10, x11, w15, x12, .Lxmm_setup_finished 1667 SETUP_REFERENCE_PARAMETER_IN_GPR w7, x10, x11, w15, x12, .Lxmm_setup_finished 1668 add x28, x28, #OFFSET_TO_FIRST_ARGUMENT_IN_STACK 1669 SETUP_REFERENCE_PARAMETERS_IN_STACK x10, x11, w15, x28, x12 1670 b .Lxmm_setup_finished 1671 1672.Lsetup_slow_path: 1673 // If the method is not static and there is one argument ('this'), we don't need to fetch the 1674 // shorty. 1675 tbnz w26, #ART_METHOD_IS_STATIC_FLAG_BIT, .Lsetup_with_shorty 1676 str w1, [xFP, x27] 1677 str w1, [xREFS, x27] 1678 cmp w15, #1 1679 b.eq .Lxmm_setup_finished 1680 1681.Lsetup_with_shorty: 1682 // TODO: Get shorty in a better way and remove below 1683 SPILL_ALL_ARGUMENTS 1684 bl NterpGetShorty 1685 // Save shorty in callee-save xIBASE. 1686 mov xIBASE, x0 1687 RESTORE_ALL_ARGUMENTS 1688 1689 // Setup pointer to inputs in FP and pointer to inputs in REFS 1690 add x10, xFP, x27 1691 add x11, xREFS, x27 1692 mov x12, #0 1693 1694 add x9, xIBASE, #1 // shorty + 1 ; ie skip return arg character 1695 tbnz w26, #ART_METHOD_IS_STATIC_FLAG_BIT, .Lhandle_static_method 1696 add x10, x10, #4 1697 add x11, x11, #4 1698 add x28, x28, #4 1699 b .Lcontinue_setup_gprs 1700.Lhandle_static_method: 1701 LOOP_OVER_SHORTY_STORING_GPRS x1, w1, x9, x12, x10, x11, .Lgpr_setup_finished 1702.Lcontinue_setup_gprs: 1703 LOOP_OVER_SHORTY_STORING_GPRS x2, w2, x9, x12, x10, x11, .Lgpr_setup_finished 1704 LOOP_OVER_SHORTY_STORING_GPRS x3, w3, x9, x12, x10, x11, .Lgpr_setup_finished 1705 LOOP_OVER_SHORTY_STORING_GPRS x4, w4, x9, x12, x10, x11, .Lgpr_setup_finished 1706 LOOP_OVER_SHORTY_STORING_GPRS x5, w5, x9, x12, x10, x11, .Lgpr_setup_finished 1707 LOOP_OVER_SHORTY_STORING_GPRS x6, w6, x9, x12, x10, x11, .Lgpr_setup_finished 1708 LOOP_OVER_SHORTY_STORING_GPRS x7, w7, x9, x12, x10, x11, .Lgpr_setup_finished 1709 LOOP_OVER_INTs x9, x12, x10, x11, x28, .Lgpr_setup_finished 1710.Lgpr_setup_finished: 1711 add x9, xIBASE, #1 // shorty + 1 ; ie skip return arg character 1712 mov x12, #0 // reset counter 1713 LOOP_OVER_SHORTY_STORING_FPS d0, s0, x9, x12, x10, .Lxmm_setup_finished 1714 LOOP_OVER_SHORTY_STORING_FPS d1, s1, x9, x12, x10, .Lxmm_setup_finished 1715 LOOP_OVER_SHORTY_STORING_FPS d2, s2, x9, x12, x10, .Lxmm_setup_finished 1716 LOOP_OVER_SHORTY_STORING_FPS d3, s3, x9, x12, x10, .Lxmm_setup_finished 1717 LOOP_OVER_SHORTY_STORING_FPS d4, s4, x9, x12, x10, .Lxmm_setup_finished 1718 LOOP_OVER_SHORTY_STORING_FPS d5, s5, x9, x12, x10, .Lxmm_setup_finished 1719 LOOP_OVER_SHORTY_STORING_FPS d6, s6, x9, x12, x10, .Lxmm_setup_finished 1720 LOOP_OVER_SHORTY_STORING_FPS d7, s7, x9, x12, x10, .Lxmm_setup_finished 1721 LOOP_OVER_FPs x9, x12, x10, x28, .Lxmm_setup_finished 1722.Lxmm_setup_finished: 1723 CFI_DEFINE_DEX_PC_WITH_OFFSET(CFI_TMP, CFI_DEX, 0) 1724 1725 // Set rIBASE 1726 adr xIBASE, artNterpAsmInstructionStart 1727 /* start executing the instruction at xPC */ 1728 START_EXECUTING_INSTRUCTIONS 1729 /* NOTE: no fallthrough */ 1730 // cfi info continues, and covers the whole nterp implementation. 1731 SIZE ExecuteNterpImpl 1732 1733%def opcode_pre(): 1734 1735%def fetch_from_thread_cache(dest_reg, miss_label): 1736 // Fetch some information from the thread cache. 1737 // Uses ip and ip2 as temporaries. 1738 add ip, xSELF, #THREAD_INTERPRETER_CACHE_OFFSET // cache address 1739 ubfx ip2, xPC, #2, #THREAD_INTERPRETER_CACHE_SIZE_LOG2 // entry index 1740 add ip, ip, ip2, lsl #4 // entry address within the cache 1741 ldp ip, ${dest_reg}, [ip] // entry key (pc) and value (offset) 1742 cmp ip, xPC 1743 b.ne ${miss_label} 1744 1745%def footer(): 1746/* 1747 * =========================================================================== 1748 * Common subroutines and data 1749 * =========================================================================== 1750 */ 1751 1752 .text 1753 .align 2 1754 1755// Enclose all code below in a symbol (which gets printed in backtraces). 1756NAME_START nterp_helper 1757 1758// Note: mterp also uses the common_* names below for helpers, but that's OK 1759// as the assembler compiled each interpreter separately. 1760common_errDivideByZero: 1761 EXPORT_PC 1762 bl art_quick_throw_div_zero 1763 1764// Expect index in w1, length in w3. 1765common_errArrayIndex: 1766 EXPORT_PC 1767 mov x0, x1 1768 mov x1, x3 1769 bl art_quick_throw_array_bounds 1770 1771common_errNullObject: 1772 EXPORT_PC 1773 bl art_quick_throw_null_pointer_exception 1774 1775NterpCommonInvokeStatic: 1776 COMMON_INVOKE_NON_RANGE is_static=1, suffix="invokeStatic" 1777 1778NterpCommonInvokeStaticRange: 1779 COMMON_INVOKE_RANGE is_static=1, suffix="invokeStatic" 1780 1781NterpCommonInvokeInstance: 1782 COMMON_INVOKE_NON_RANGE suffix="invokeInstance" 1783 1784NterpCommonInvokeInstanceRange: 1785 COMMON_INVOKE_RANGE suffix="invokeInstance" 1786 1787NterpCommonInvokeInterface: 1788 COMMON_INVOKE_NON_RANGE is_interface=1, suffix="invokeInterface" 1789 1790NterpCommonInvokeInterfaceRange: 1791 COMMON_INVOKE_RANGE is_interface=1, suffix="invokeInterface" 1792 1793NterpCommonInvokePolymorphic: 1794 COMMON_INVOKE_NON_RANGE is_polymorphic=1, suffix="invokePolymorphic" 1795 1796NterpCommonInvokePolymorphicRange: 1797 COMMON_INVOKE_RANGE is_polymorphic=1, suffix="invokePolymorphic" 1798 1799NterpCommonInvokeCustom: 1800 COMMON_INVOKE_NON_RANGE is_static=1, is_custom=1, suffix="invokeCustom" 1801 1802NterpCommonInvokeCustomRange: 1803 COMMON_INVOKE_RANGE is_static=1, is_custom=1, suffix="invokeCustom" 1804 1805NterpHandleStringInit: 1806 COMMON_INVOKE_NON_RANGE is_string_init=1, suffix="stringInit" 1807 1808NterpHandleStringInitRange: 1809 COMMON_INVOKE_RANGE is_string_init=1, suffix="stringInit" 1810 1811NterpHandleHotnessOverflow: 1812 CHECK_AND_UPDATE_SHARED_MEMORY_METHOD if_hot=1f, if_not_hot=5f 18131: 1814 mov x1, xPC 1815 mov x2, xFP 1816 bl nterp_hot_method 1817 cbnz x0, 3f 18182: 1819 FETCH wINST, 0 // load wINST 1820 GET_INST_OPCODE ip // extract opcode from wINST 1821 GOTO_OPCODE ip // jump to next instruction 18223: 1823 // Drop the current frame. 1824 ldr ip, [xREFS, #-8] 1825 mov sp, ip 1826 .cfi_def_cfa sp, CALLEE_SAVES_SIZE 1827 1828 // The transition frame of type SaveAllCalleeSaves saves x19 and x20, 1829 // but not managed ABI. So we need to restore callee-saves of the nterp frame, 1830 // and save managed ABI callee saves, which will be restored by the callee upon 1831 // return. 1832 RESTORE_ALL_CALLEE_SAVES 1833 INCREASE_FRAME ((CALLEE_SAVES_SIZE) - 16) 1834 1835 // FP callee-saves 1836 stp d8, d9, [sp, #0] 1837 stp d10, d11, [sp, #16] 1838 stp d12, d13, [sp, #32] 1839 stp d14, d15, [sp, #48] 1840 1841 // GP callee-saves. 1842 SAVE_TWO_REGS x21, x22, 64 1843 SAVE_TWO_REGS x23, x24, 80 1844 SAVE_TWO_REGS x25, x26, 96 1845 SAVE_TWO_REGS x27, x28, 112 1846 SAVE_TWO_REGS x29, lr, 128 1847 1848 // Setup the new frame 1849 ldr x1, [x0, #OSR_DATA_FRAME_SIZE] 1850 // Given stack size contains all callee saved registers, remove them. 1851 sub x1, x1, #(CALLEE_SAVES_SIZE - 16) 1852 1853 // We know x1 cannot be 0, as it at least contains the ArtMethod. 1854 1855 // Remember CFA in a callee-save register. 1856 mov xINST, sp 1857 .cfi_def_cfa_register xINST 1858 1859 sub sp, sp, x1 1860 1861 add x2, x0, #OSR_DATA_MEMORY 18624: 1863 sub x1, x1, #8 1864 ldr ip, [x2, x1] 1865 str ip, [sp, x1] 1866 cbnz x1, 4b 1867 1868 // Fetch the native PC to jump to and save it in a callee-save register. 1869 ldr xFP, [x0, #OSR_DATA_NATIVE_PC] 1870 1871 // Free the memory holding OSR Data. 1872 bl free 1873 1874 // Jump to the compiled code. 1875 br xFP 18765: 1877 DO_SUSPEND_CHECK continue_label=2b 1878 b 2b 1879 1880// This is the logical end of ExecuteNterpImpl, where the frame info applies. 1881// EndExecuteNterpImpl includes the methods below as we want the runtime to 1882// see them as part of the Nterp PCs. 1883.cfi_endproc 1884 1885nterp_to_nterp_static_non_range: 1886 .cfi_startproc 1887 SETUP_STACK_FOR_INVOKE 1888 SETUP_NON_RANGE_ARGUMENTS_AND_EXECUTE is_static=1, is_string_init=0 1889 .cfi_endproc 1890 1891nterp_to_nterp_string_init_non_range: 1892 .cfi_startproc 1893 SETUP_STACK_FOR_INVOKE 1894 SETUP_NON_RANGE_ARGUMENTS_AND_EXECUTE is_static=0, is_string_init=1 1895 .cfi_endproc 1896 1897nterp_to_nterp_instance_non_range: 1898 .cfi_startproc 1899 SETUP_STACK_FOR_INVOKE 1900 SETUP_NON_RANGE_ARGUMENTS_AND_EXECUTE is_static=0, is_string_init=0 1901 .cfi_endproc 1902 1903nterp_to_nterp_static_range: 1904 .cfi_startproc 1905 SETUP_STACK_FOR_INVOKE 1906 SETUP_RANGE_ARGUMENTS_AND_EXECUTE is_static=1 1907 .cfi_endproc 1908 1909nterp_to_nterp_instance_range: 1910 .cfi_startproc 1911 SETUP_STACK_FOR_INVOKE 1912 SETUP_RANGE_ARGUMENTS_AND_EXECUTE is_static=0 1913 .cfi_endproc 1914 1915nterp_to_nterp_string_init_range: 1916 .cfi_startproc 1917 SETUP_STACK_FOR_INVOKE 1918 SETUP_RANGE_ARGUMENTS_AND_EXECUTE is_static=0, is_string_init=1 1919 .cfi_endproc 1920 1921NAME_END nterp_helper 1922 1923// This is the end of PCs contained by the OatQuickMethodHeader created for the interpreter 1924// entry point. 1925 .type EndExecuteNterpImpl, #function 1926 .hidden EndExecuteNterpImpl 1927 .global EndExecuteNterpImpl 1928EndExecuteNterpImpl: 1929 1930// Entrypoints into runtime. 1931NTERP_TRAMPOLINE nterp_get_static_field, NterpGetStaticField 1932NTERP_TRAMPOLINE nterp_get_instance_field_offset, NterpGetInstanceFieldOffset 1933NTERP_TRAMPOLINE nterp_filled_new_array, NterpFilledNewArray 1934NTERP_TRAMPOLINE nterp_filled_new_array_range, NterpFilledNewArrayRange 1935NTERP_TRAMPOLINE nterp_get_class, NterpGetClass 1936NTERP_TRAMPOLINE nterp_allocate_object, NterpAllocateObject 1937NTERP_TRAMPOLINE nterp_get_method, NterpGetMethod 1938NTERP_TRAMPOLINE nterp_hot_method, NterpHotMethod 1939NTERP_TRAMPOLINE nterp_load_object, NterpLoadObject 1940 1941ENTRY nterp_deliver_pending_exception 1942 DELIVER_PENDING_EXCEPTION 1943END nterp_deliver_pending_exception 1944 1945// gen_mterp.py will inline the following definitions 1946// within [ExecuteNterpImpl, EndExecuteNterpImpl). 1947%def instruction_end(): 1948 1949 .type artNterpAsmInstructionEnd, #function 1950 .hidden artNterpAsmInstructionEnd 1951 .global artNterpAsmInstructionEnd 1952artNterpAsmInstructionEnd: 1953 // artNterpAsmInstructionEnd is used as landing pad for exception handling. 1954 FETCH_INST 1955 GET_INST_OPCODE ip 1956 GOTO_OPCODE ip 1957 1958%def instruction_start(): 1959 1960 .type artNterpAsmInstructionStart, #function 1961 .hidden artNterpAsmInstructionStart 1962 .global artNterpAsmInstructionStart 1963artNterpAsmInstructionStart = .L_op_nop 1964 .text 1965 1966%def opcode_name_prefix(): 1967% return "nterp_" 1968%def opcode_start(): 1969 NAME_START nterp_${opcode} 1970 # Explicitly restore CFA, just in case the previous opcode clobbered it (by .cfi_def_*). 1971 CFI_DEF_CFA_BREG_PLUS_UCONST CFI_REFS, -8, CALLEE_SAVES_SIZE 1972%def opcode_end(): 1973 NAME_END nterp_${opcode} 1974 // Advance to the end of this handler. Causes error if we are past that point. 1975 .org nterp_${opcode} + NTERP_HANDLER_SIZE // ${opcode} handler is too big! 1976%def opcode_slow_path_start(name): 1977 NAME_START ${name} 1978%def opcode_slow_path_end(name): 1979 NAME_END ${name} 1980