Emit reference load

src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp

@@ -2969,6 +2969,48 @@ void LIR_Assembler::emit_profile_inline_type(LIR_OpProfileInlineType* op) {
   __ bind(not_inline_type);
 }
 
+void LIR_Assembler::emit_profile_multiple_array_types(LIR_OpProfileMultipleArrayTypes* op) {
+  Register array = op->array()->as_pointer_register();
+  Register tmp1 = op->tmp1()->as_pointer_register();
+  Register tmp2 = op->tmp2()->as_pointer_register();
+  ciMethodData* md = op->md();
+
+  Label not_flat, done;
+  __ test_non_flat_array_oop (array, tmp1, not_flat);
+
+  Register klass = tmp1;
+  __ load_klass(klass, array);
+
+  Register mdo = tmp2;
+  __ mov_metadata(mdo, md->constant_encoding());
+
+  int mdp_offset = md->byte_offset_of_slot(op->load(), in_ByteSize(0));
+  __ profile_array_type_at_load(tmp1, mdo, mdp_offset);
+
+  __ b(done);
+  __ bind(not_flat);
+  __ mov_metadata(mdo, md->constant_encoding());
+
+  Label null_free;
+
+  __ test_null_free_array_oop(array, tmp1, null_free);
+
+  {
+    Address counter_addr(mdo, md->byte_offset_of_slot(op->load(), ArrayLoadData::not_flat_nullable_count_offset()));
+    __ addptr(counter_addr, DataLayout::counter_increment);
+  }
+
+  __ b(done);
+  __ bind(null_free);
+
+  {
+    Address counter_addr(mdo, md->byte_offset_of_slot(op->load(), ArrayLoadData::not_flat_null_free_count_offset()));
+    __ addptr(counter_addr, DataLayout::counter_increment);
+  }
+
+  __ bind(done);
+}
+
 void LIR_Assembler::align_backward_branch_target() {
 }
 

src/hotspot/cpu/aarch64/interp_masm_aarch64.cpp

@@ -1202,44 +1202,70 @@ void InterpreterMacroAssembler::profile_switch_case(Register index,
   }
 }
 
-template <class ArrayData> void InterpreterMacroAssembler::profile_array_type(Register mdp,
-                                                                              Register array,
-                                                                              Register tmp) {
+void InterpreterMacroAssembler::profile_array_type(Register mdp,
+                                                   Register array,
+                                                   Register tmp) {
   if (ProfileInterpreter) {
     Label profile_continue;
 
     // If no method data exists, go to profile_continue.
     test_method_data_pointer(mdp, profile_continue);
 
     mov(tmp, array);
-    profile_obj_type(tmp, Address(mdp, in_bytes(ArrayData::array_offset())));
+    profile_obj_type(tmp, Address(mdp, in_bytes(ArrayStoreData::array_offset())));
 
     Label not_flat;
     test_non_flat_array_oop(array, tmp, not_flat);
 
-    set_mdp_flag_at(mdp, ArrayData::flat_array_byte_constant());
+    set_mdp_flag_at(mdp, ArrayStoreData::flat_array_byte_constant());
 
     bind(not_flat);
 
     Label not_null_free;
     test_non_null_free_array_oop(array, tmp, not_null_free);
 
-    set_mdp_flag_at(mdp, ArrayData::null_free_array_byte_constant());
+    set_mdp_flag_at(mdp, ArrayStoreData::null_free_array_byte_constant());
 
     bind(not_null_free);
 
     bind(profile_continue);
   }
 }
 
-template void InterpreterMacroAssembler::profile_array_type<ArrayLoadData>(Register mdp,
-                                                                           Register array,
-                                                                           Register tmp);
-template void InterpreterMacroAssembler::profile_array_type<ArrayStoreData>(Register mdp,
-                                                                            Register array,
-                                                                            Register tmp);
+void InterpreterMacroAssembler::profile_multiple_array_types(Register mdp,
+                                                             Register array,
+                                                             Register tmp) {
+  if (ProfileInterpreter) {
+    Label profile_continue;
+
+    // If no method data exists, go to profile_continue.
+    test_method_data_pointer(mdp, profile_continue);
+
+    Label not_flat;
+    test_non_flat_array_oop(array, tmp, not_flat);
+
+    load_klass(tmp, array);
+    profile_array_type_at_load(tmp, mdp, 0);
+
+    b(profile_continue);
+    bind(not_flat);
+
+    Label not_null_free;
+    test_non_null_free_array_oop(array, tmp, not_null_free);
+
+    increment_mdp_data_at(mdp, in_bytes(ArrayLoadData::not_flat_null_free_count_offset()));
+
+    b(profile_continue);
+
+    bind(not_null_free);
+
+    increment_mdp_data_at(mdp, in_bytes(ArrayLoadData::not_flat_nullable_count_offset()));
+
+    bind(profile_continue);
+  }
+}
 
-void InterpreterMacroAssembler::profile_multiple_element_types(Register mdp, Register element, Register tmp, const Register tmp2) {
+void InterpreterMacroAssembler::profile_multiple_element_types(Register mdp, Register element, Register tmp) {
   if (ProfileInterpreter) {
     Label profile_continue;
 

src/hotspot/cpu/aarch64/interp_masm_aarch64.hpp

@@ -303,8 +303,9 @@ class InterpreterMacroAssembler: public MacroAssembler {
   void profile_switch_default(Register mdp);
   void profile_switch_case(Register index_in_scratch, Register mdp,
                            Register scratch2);
-  template <class ArrayData> void profile_array_type(Register mdp, Register array, Register tmp);
-  void profile_multiple_element_types(Register mdp, Register element, Register tmp, Register tmp2);
+  void profile_array_type(Register mdp, Register array, Register tmp);
+  void profile_multiple_array_types(Register mdp, Register array, Register tmp);
+  void profile_multiple_element_types(Register mdp, Register element, Register tmp);
   void profile_element_type(Register mdp, Register element, Register tmp);
   void profile_acmp(Register mdp, Register left, Register right, Register tmp);
 

src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp

@@ -2112,56 +2112,32 @@ Address MacroAssembler::argument_address(RegisterOrConstant arg_slot,
   }
 }
 
-// Handle the receiver type profile update given the "recv" klass.
-//
-// Normally updates the ReceiverData (RD) that starts at "mdp" + "mdp_offset".
-// If there are no matching or claimable receiver entries in RD, updates
-// the polymorphic counter.
-//
-// This code expected to run by either the interpreter or JIT-ed code, without
-// extra synchronization. For safety, receiver cells are claimed atomically, which
-// avoids grossly misrepresenting the profiles under concurrent updates. For speed,
-// counter updates are not atomic.
-//
-void MacroAssembler::profile_receiver_type(Register recv, Register mdp, int mdp_offset) {
-  assert_different_registers(recv, mdp, rscratch1, rscratch2);
-
-  int base_receiver_offset   = in_bytes(ReceiverTypeData::receiver_offset(0));
-  int end_receiver_offset    = in_bytes(ReceiverTypeData::receiver_offset(ReceiverTypeData::row_limit()));
-  int poly_count_offset      = in_bytes(CounterData::count_offset());
-  int receiver_step          = in_bytes(ReceiverTypeData::receiver_offset(1)) - base_receiver_offset;
-  int receiver_to_count_step = in_bytes(ReceiverTypeData::receiver_count_offset(0)) - base_receiver_offset;
+void MacroAssembler::profile_receiver_type_helper(Register recv, Register mdp, Label &L_found_recv, int mdp_offset, int base, uint row_limit) {
+  int base_receiver_offset   = in_bytes(MegamorphicTypeData::receiver_offset(base, 0));
+  int end_receiver_offset    = in_bytes(MegamorphicTypeData::receiver_offset(base, row_limit));
+  int receiver_step          = in_bytes(MegamorphicTypeData::receiver_offset(base, 1)) - base_receiver_offset;
+  int receiver_to_count_step = in_bytes(MegamorphicTypeData::receiver_count_offset(base, 0)) - base_receiver_offset;
 
-  // Adjust for MDP offsets.
   base_receiver_offset += mdp_offset;
   end_receiver_offset  += mdp_offset;
-  poly_count_offset    += mdp_offset;
 
 #ifdef ASSERT
   // We are about to walk the MDO slots without asking for offsets.
   // Check that our math hits all the right spots.
   for (uint c = 0; c < ReceiverTypeData::row_limit(); c++) {
-    int real_recv_offset  = mdp_offset + in_bytes(ReceiverTypeData::receiver_offset(c));
-    int real_count_offset = mdp_offset + in_bytes(ReceiverTypeData::receiver_count_offset(c));
+    int real_recv_offset  = mdp_offset + in_bytes(MegamorphicTypeData::receiver_offset(base, c));
+    int real_count_offset = mdp_offset + in_bytes(MegamorphicTypeData::receiver_count_offset(base, c));
     int offset = base_receiver_offset + receiver_step*c;
     int count_offset = offset + receiver_to_count_step;
-    assert(offset == real_recv_offset, "receiver slot math");
+    assert(offset== real_recv_offset, "receiver slot math");
     assert(count_offset == real_count_offset, "receiver count math");
   }
-  int real_poly_count_offset = mdp_offset + in_bytes(CounterData::count_offset());
-  assert(poly_count_offset == real_poly_count_offset, "poly counter math");
 #endif
 
-  // Corner case: no profile table. Increment poly counter and exit.
-  if (ReceiverTypeData::row_limit() == 0) {
-    increment(Address(mdp, poly_count_offset), DataLayout::counter_increment);
-    return;
-  }
-
   Register offset = rscratch2;
 
   Label L_loop_search_receiver, L_loop_search_empty;
-  Label L_restart, L_found_recv, L_found_empty, L_count_update;
+  Label L_restart, L_found_empty, L_polymorphic;
 
   // The code here recognizes three major cases:
   //   A. Fastest: receiver found in the table
@@ -2214,28 +2190,24 @@ void MacroAssembler::profile_receiver_type(Register recv, Register mdp, int mdp_
   // Fastest: receiver is already installed
   mov(offset, base_receiver_offset);
   bind(L_loop_search_receiver);
-    ldr(rscratch1, Address(mdp, offset));
-    cmp(rscratch1, recv);
-    br(Assembler::EQ, L_found_recv);
+  ldr(rscratch1, Address(mdp, offset));
+  cmp(rscratch1, recv);
+  br(Assembler::EQ, L_found_recv);
   add(offset, offset, receiver_step);
   sub(rscratch1, offset, end_receiver_offset);
   cbnz(rscratch1, L_loop_search_receiver);
 
   // Fast: no receiver, but profile is not full
   mov(offset, base_receiver_offset);
   bind(L_loop_search_empty);
-    ldr(rscratch1, Address(mdp, offset));
-    cbz(rscratch1, L_found_empty);
+  ldr(rscratch1, Address(mdp, offset));
+  cbz(rscratch1, L_found_empty);
   add(offset, offset, receiver_step);
   sub(rscratch1, offset, end_receiver_offset);
   cbnz(rscratch1, L_loop_search_empty);
+  b(L_polymorphic);
 
-  // Slow: Receiver is not found and table is full.
-  // Increment polymorphic counter instead of receiver slot.
-  mov(offset, poly_count_offset);
-  b(L_count_update);
-
-  // Slowest: try to install receiver
+  // Slow: try to install receiver
   bind(L_found_empty);
 
   // Atomically swing receiver slot: null -> recv.
@@ -2254,15 +2226,120 @@ void MacroAssembler::profile_receiver_type(Register recv, Register mdp, int mdp_
   // and just restart the search from the beginning.
   b(L_restart);
 
+  // Counter updates:
+
+  // Increment polymorphic counter instead of receiver slot.
+  bind(L_polymorphic);
+}
+
+// Handle the receiver type profile update given the "recv" klass.
+//
+// Normally updates the ReceiverData (RD) that starts at "mdp" + "mdp_offset".
+// If there are no matching or claimable receiver entries in RD, updates
+// the polymorphic counter.
+//
+// This code expected to run by either the interpreter or JIT-ed code, without
+// extra synchronization. For safety, receiver cells are claimed atomically, which
+// avoids grossly misrepresenting the profiles under concurrent updates. For speed,
+// counter updates are not atomic.
+//
+void MacroAssembler::profile_receiver_type(Register recv, Register mdp, int mdp_offset) {
+  assert_different_registers(recv, mdp, rscratch1, rscratch2);
+
+  int poly_count_offset      = in_bytes(ReceiverTypeData::count_offset());
+  int base_receiver_offset   = in_bytes(ReceiverTypeData::receiver_offset(0));
+  int receiver_to_count_step = in_bytes(ReceiverTypeData::receiver_count_offset(0)) - base_receiver_offset;
+
+  // Adjust for MDP offsets.
+  poly_count_offset    += mdp_offset;
+
+#ifdef ASSERT
+  int real_poly_count_offset = mdp_offset + in_bytes(ReceiverTypeData::count_offset());
+  assert(poly_count_offset == real_poly_count_offset, "poly counter math");
+#endif
+
+  // Corner case: no profile table. Increment poly counter and exit.
+  if (ReceiverTypeData::row_limit() == 0) {
+    increment(Address(mdp, poly_count_offset), DataLayout::counter_increment);
+    return;
+  }
+
+  Label L_found_recv;
+  profile_receiver_type_helper(recv, mdp, L_found_recv, mdp_offset, ReceiverTypeData::base_of_megamorphic_type_data(), ReceiverTypeData::row_limit());
+  // Increment polymorphic counter instead of receiver slot.
+  Register offset = rscratch2;
+  mov(offset, poly_count_offset);
+  Label L_count_update;
+  b(L_count_update);
+
   // Found a receiver, convert its slot offset to corresponding count offset.
   bind(L_found_recv);
   add(offset, offset, receiver_to_count_step);
 
-  // Finally, update the counter
   bind(L_count_update);
   increment(Address(mdp, offset), DataLayout::counter_increment);
 }
 
+void MacroAssembler::profile_array_type_at_load(Register recv, Register mdp, int mdp_offset) {
+  int base_receiver_offset                  = in_bytes(ArrayLoadData::receiver_offset(0));
+  int receiver_to_count_step                = in_bytes(ArrayLoadData::receiver_count_offset(0)) - base_receiver_offset;
+  int flat_nullable_count_offset            = in_bytes(ArrayLoadData::flat_nullable_count_offset());
+  int flat_nullfree_atomic_count_offset     = in_bytes(ArrayLoadData::flat_nullfree_atomic_count_offset());
+  int flat_nullfree_not_atomic_count_offset = in_bytes(ArrayLoadData::flat_nullfree_not_atomic_count_offset());
+
+  // Adjust for MDP offsets. Slots are pointer-sized, so is the global offset.
+  flat_nullable_count_offset            += mdp_offset;
+  flat_nullfree_atomic_count_offset     += mdp_offset;
+  flat_nullfree_not_atomic_count_offset += mdp_offset;
+
+#ifdef ASSERT
+  int real_flat_nullable_count_offset = mdp_offset + in_bytes(ArrayLoadData::flat_nullable_count_offset());
+  assert(flat_nullable_count_offset == real_flat_nullable_count_offset, "poly counter math");
+  int real_flat_nullfree_atomic_count_offset = mdp_offset + in_bytes(ArrayLoadData::flat_nullfree_atomic_count_offset());
+  assert(flat_nullfree_atomic_count_offset == real_flat_nullfree_atomic_count_offset, "poly counter math");
+  int real_flat_nullfree_not_atomic_count_offset = mdp_offset + in_bytes(ArrayLoadData::flat_nullfree_not_atomic_count_offset());
+  assert(flat_nullfree_not_atomic_count_offset == real_flat_nullfree_not_atomic_count_offset, "poly counter math");
+#endif
+
+  Label L_found_recv;
+  // Corner case: no profile table. Increment poly counter and exit.
+  if (ReceiverTypeData::row_limit() != 0) {
+    profile_receiver_type_helper(recv, mdp, L_found_recv, mdp_offset, ArrayLoadData::base_of_megamorphic_type_data(), ArrayLoadData::row_limit());
+  }
+
+  Register offset = rscratch2;
+  int layout_kind_offset = in_bytes(FlatArrayKlass::layout_kind_offset());
+  Label null_free_non_atomic, null_free_atomic, nullable_atomic_flat, failure, L_count_update;
+  ldrw(rscratch1, Address(recv, layout_kind_offset));
+  cmpw(rscratch1, (int)LayoutKind::NULL_FREE_ATOMIC_FLAT);
+  br(NE, null_free_non_atomic);
+  mov(offset, flat_nullfree_atomic_count_offset);
+
+  b(L_count_update);
+  bind(null_free_non_atomic);
+  ldrw(rscratch1, Address(recv, layout_kind_offset));
+  cmpw(rscratch1, (int)LayoutKind::NULL_FREE_NON_ATOMIC_FLAT);
+  br(NE, nullable_atomic_flat);
+  mov(offset, flat_nullfree_not_atomic_count_offset);
+
+  b(L_count_update);
+  bind(nullable_atomic_flat);
+  ldrw(rscratch1, Address(recv, layout_kind_offset));
+  cmpw(rscratch1, (int)LayoutKind::NULLABLE_ATOMIC_FLAT);
+  br(NE, failure);
+  mov(offset, flat_nullable_count_offset);
+
+  b(L_count_update);
+  bind(failure);
+  stop("unexpected flat array");
+
+  // Found a receiver, convert its slot offset to corresponding count offset.
+  bind(L_found_recv);
+  add(offset, offset, receiver_to_count_step);
+
+  bind(L_count_update);
+  increment(Address(mdp, offset), DataLayout::counter_increment);
+}
 
 void MacroAssembler::call_VM_leaf_base(address entry_point,
                                        int number_of_arguments,

src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp

@@ -1177,7 +1177,11 @@ class MacroAssembler: public Assembler {
 
   Address argument_address(RegisterOrConstant arg_slot, int extra_slot_offset = 0);
 
+  void profile_receiver_type_helper(Register recv, Register mdp,
+                                    Label &L_found_recv, int mdp_offset, int base, uint row_limit);
+
   void profile_receiver_type(Register recv, Register mdp, int mdp_offset);
+  void profile_array_type_at_load(Register recv, Register mdp, int mdp_offset);
 
   void verify_sve_vector_length(Register tmp = rscratch1);
   void reinitialize_ptrue() {