diff --git a/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp index 1fc0f43ed6c..ea1da15ba87 100644 --- a/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp @@ -2969,6 +2969,48 @@ void LIR_Assembler::emit_profile_inline_type(LIR_OpProfileInlineType* op) { __ bind(not_inline_type); } +void LIR_Assembler::emit_profile_multiple_array_types(LIR_OpProfileMultipleArrayTypes* op) { + Register array = op->array()->as_pointer_register(); + Register tmp1 = op->tmp1()->as_pointer_register(); + Register tmp2 = op->tmp2()->as_pointer_register(); + ciMethodData* md = op->md(); + + Label not_flat, done; + __ test_non_flat_array_oop (array, tmp1, not_flat); + + Register klass = tmp1; + __ load_klass(klass, array); + + Register mdo = tmp2; + __ mov_metadata(mdo, md->constant_encoding()); + + int mdp_offset = md->byte_offset_of_slot(op->load(), in_ByteSize(0)); + __ profile_array_type_at_load(tmp1, mdo, mdp_offset); + + __ b(done); + __ bind(not_flat); + __ mov_metadata(mdo, md->constant_encoding()); + + Label null_free; + + __ test_null_free_array_oop(array, tmp1, null_free); + + { + Address counter_addr(mdo, md->byte_offset_of_slot(op->load(), ArrayLoadData::not_flat_nullable_count_offset())); + __ addptr(counter_addr, DataLayout::counter_increment); + } + + __ b(done); + __ bind(null_free); + + { + Address counter_addr(mdo, md->byte_offset_of_slot(op->load(), ArrayLoadData::not_flat_null_free_count_offset())); + __ addptr(counter_addr, DataLayout::counter_increment); + } + + __ bind(done); +} + void LIR_Assembler::align_backward_branch_target() { } diff --git a/src/hotspot/cpu/aarch64/interp_masm_aarch64.cpp b/src/hotspot/cpu/aarch64/interp_masm_aarch64.cpp index 1b16ec25ca7..57048e1937e 100644 --- a/src/hotspot/cpu/aarch64/interp_masm_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/interp_masm_aarch64.cpp @@ -1202,9 +1202,9 @@ void InterpreterMacroAssembler::profile_switch_case(Register index, } } -template void InterpreterMacroAssembler::profile_array_type(Register mdp, - Register array, - Register tmp) { +void InterpreterMacroAssembler::profile_array_type(Register mdp, + Register array, + Register tmp) { if (ProfileInterpreter) { Label profile_continue; @@ -1212,19 +1212,19 @@ template void InterpreterMacroAssembler::profile_array_type(Re test_method_data_pointer(mdp, profile_continue); mov(tmp, array); - profile_obj_type(tmp, Address(mdp, in_bytes(ArrayData::array_offset()))); + profile_obj_type(tmp, Address(mdp, in_bytes(ArrayStoreData::array_offset()))); Label not_flat; test_non_flat_array_oop(array, tmp, not_flat); - set_mdp_flag_at(mdp, ArrayData::flat_array_byte_constant()); + set_mdp_flag_at(mdp, ArrayStoreData::flat_array_byte_constant()); bind(not_flat); Label not_null_free; test_non_null_free_array_oop(array, tmp, not_null_free); - set_mdp_flag_at(mdp, ArrayData::null_free_array_byte_constant()); + set_mdp_flag_at(mdp, ArrayStoreData::null_free_array_byte_constant()); bind(not_null_free); @@ -1232,14 +1232,40 @@ template void InterpreterMacroAssembler::profile_array_type(Re } } -template void InterpreterMacroAssembler::profile_array_type(Register mdp, - Register array, - Register tmp); -template void InterpreterMacroAssembler::profile_array_type(Register mdp, - Register array, - Register tmp); +void InterpreterMacroAssembler::profile_multiple_array_types(Register mdp, + Register array, + Register tmp) { + if (ProfileInterpreter) { + Label profile_continue; + + // If no method data exists, go to profile_continue. + test_method_data_pointer(mdp, profile_continue); + + Label not_flat; + test_non_flat_array_oop(array, tmp, not_flat); + + load_klass(tmp, array); + profile_array_type_at_load(tmp, mdp, 0); + + b(profile_continue); + bind(not_flat); + + Label not_null_free; + test_non_null_free_array_oop(array, tmp, not_null_free); + + increment_mdp_data_at(mdp, in_bytes(ArrayLoadData::not_flat_null_free_count_offset())); + + b(profile_continue); + + bind(not_null_free); + + increment_mdp_data_at(mdp, in_bytes(ArrayLoadData::not_flat_nullable_count_offset())); + + bind(profile_continue); + } +} -void InterpreterMacroAssembler::profile_multiple_element_types(Register mdp, Register element, Register tmp, const Register tmp2) { +void InterpreterMacroAssembler::profile_multiple_element_types(Register mdp, Register element, Register tmp) { if (ProfileInterpreter) { Label profile_continue; diff --git a/src/hotspot/cpu/aarch64/interp_masm_aarch64.hpp b/src/hotspot/cpu/aarch64/interp_masm_aarch64.hpp index d72137a0944..12f359de284 100644 --- a/src/hotspot/cpu/aarch64/interp_masm_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/interp_masm_aarch64.hpp @@ -303,8 +303,9 @@ class InterpreterMacroAssembler: public MacroAssembler { void profile_switch_default(Register mdp); void profile_switch_case(Register index_in_scratch, Register mdp, Register scratch2); - template void profile_array_type(Register mdp, Register array, Register tmp); - void profile_multiple_element_types(Register mdp, Register element, Register tmp, Register tmp2); + void profile_array_type(Register mdp, Register array, Register tmp); + void profile_multiple_array_types(Register mdp, Register array, Register tmp); + void profile_multiple_element_types(Register mdp, Register element, Register tmp); void profile_element_type(Register mdp, Register element, Register tmp); void profile_acmp(Register mdp, Register left, Register right, Register tmp); diff --git a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp index 4a7ebeae504..50bb22fd937 100644 --- a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp @@ -2112,56 +2112,32 @@ Address MacroAssembler::argument_address(RegisterOrConstant arg_slot, } } -// Handle the receiver type profile update given the "recv" klass. -// -// Normally updates the ReceiverData (RD) that starts at "mdp" + "mdp_offset". -// If there are no matching or claimable receiver entries in RD, updates -// the polymorphic counter. -// -// This code expected to run by either the interpreter or JIT-ed code, without -// extra synchronization. For safety, receiver cells are claimed atomically, which -// avoids grossly misrepresenting the profiles under concurrent updates. For speed, -// counter updates are not atomic. -// -void MacroAssembler::profile_receiver_type(Register recv, Register mdp, int mdp_offset) { - assert_different_registers(recv, mdp, rscratch1, rscratch2); - - int base_receiver_offset = in_bytes(ReceiverTypeData::receiver_offset(0)); - int end_receiver_offset = in_bytes(ReceiverTypeData::receiver_offset(ReceiverTypeData::row_limit())); - int poly_count_offset = in_bytes(CounterData::count_offset()); - int receiver_step = in_bytes(ReceiverTypeData::receiver_offset(1)) - base_receiver_offset; - int receiver_to_count_step = in_bytes(ReceiverTypeData::receiver_count_offset(0)) - base_receiver_offset; +void MacroAssembler::profile_receiver_type_helper(Register recv, Register mdp, Label &L_found_recv, int mdp_offset, int base, uint row_limit) { + int base_receiver_offset = in_bytes(MegamorphicTypeData::receiver_offset(base, 0)); + int end_receiver_offset = in_bytes(MegamorphicTypeData::receiver_offset(base, row_limit)); + int receiver_step = in_bytes(MegamorphicTypeData::receiver_offset(base, 1)) - base_receiver_offset; + int receiver_to_count_step = in_bytes(MegamorphicTypeData::receiver_count_offset(base, 0)) - base_receiver_offset; - // Adjust for MDP offsets. base_receiver_offset += mdp_offset; end_receiver_offset += mdp_offset; - poly_count_offset += mdp_offset; #ifdef ASSERT // We are about to walk the MDO slots without asking for offsets. // Check that our math hits all the right spots. for (uint c = 0; c < ReceiverTypeData::row_limit(); c++) { - int real_recv_offset = mdp_offset + in_bytes(ReceiverTypeData::receiver_offset(c)); - int real_count_offset = mdp_offset + in_bytes(ReceiverTypeData::receiver_count_offset(c)); + int real_recv_offset = mdp_offset + in_bytes(MegamorphicTypeData::receiver_offset(base, c)); + int real_count_offset = mdp_offset + in_bytes(MegamorphicTypeData::receiver_count_offset(base, c)); int offset = base_receiver_offset + receiver_step*c; int count_offset = offset + receiver_to_count_step; - assert(offset == real_recv_offset, "receiver slot math"); + assert(offset== real_recv_offset, "receiver slot math"); assert(count_offset == real_count_offset, "receiver count math"); } - int real_poly_count_offset = mdp_offset + in_bytes(CounterData::count_offset()); - assert(poly_count_offset == real_poly_count_offset, "poly counter math"); #endif - // Corner case: no profile table. Increment poly counter and exit. - if (ReceiverTypeData::row_limit() == 0) { - increment(Address(mdp, poly_count_offset), DataLayout::counter_increment); - return; - } - Register offset = rscratch2; Label L_loop_search_receiver, L_loop_search_empty; - Label L_restart, L_found_recv, L_found_empty, L_count_update; + Label L_restart, L_found_empty, L_polymorphic; // The code here recognizes three major cases: // A. Fastest: receiver found in the table @@ -2214,9 +2190,9 @@ void MacroAssembler::profile_receiver_type(Register recv, Register mdp, int mdp_ // Fastest: receiver is already installed mov(offset, base_receiver_offset); bind(L_loop_search_receiver); - ldr(rscratch1, Address(mdp, offset)); - cmp(rscratch1, recv); - br(Assembler::EQ, L_found_recv); + ldr(rscratch1, Address(mdp, offset)); + cmp(rscratch1, recv); + br(Assembler::EQ, L_found_recv); add(offset, offset, receiver_step); sub(rscratch1, offset, end_receiver_offset); cbnz(rscratch1, L_loop_search_receiver); @@ -2224,18 +2200,14 @@ void MacroAssembler::profile_receiver_type(Register recv, Register mdp, int mdp_ // Fast: no receiver, but profile is not full mov(offset, base_receiver_offset); bind(L_loop_search_empty); - ldr(rscratch1, Address(mdp, offset)); - cbz(rscratch1, L_found_empty); + ldr(rscratch1, Address(mdp, offset)); + cbz(rscratch1, L_found_empty); add(offset, offset, receiver_step); sub(rscratch1, offset, end_receiver_offset); cbnz(rscratch1, L_loop_search_empty); + b(L_polymorphic); - // Slow: Receiver is not found and table is full. - // Increment polymorphic counter instead of receiver slot. - mov(offset, poly_count_offset); - b(L_count_update); - - // Slowest: try to install receiver + // Slow: try to install receiver bind(L_found_empty); // Atomically swing receiver slot: null -> recv. @@ -2254,15 +2226,120 @@ void MacroAssembler::profile_receiver_type(Register recv, Register mdp, int mdp_ // and just restart the search from the beginning. b(L_restart); + // Counter updates: + + // Increment polymorphic counter instead of receiver slot. + bind(L_polymorphic); +} + +// Handle the receiver type profile update given the "recv" klass. +// +// Normally updates the ReceiverData (RD) that starts at "mdp" + "mdp_offset". +// If there are no matching or claimable receiver entries in RD, updates +// the polymorphic counter. +// +// This code expected to run by either the interpreter or JIT-ed code, without +// extra synchronization. For safety, receiver cells are claimed atomically, which +// avoids grossly misrepresenting the profiles under concurrent updates. For speed, +// counter updates are not atomic. +// +void MacroAssembler::profile_receiver_type(Register recv, Register mdp, int mdp_offset) { + assert_different_registers(recv, mdp, rscratch1, rscratch2); + + int poly_count_offset = in_bytes(ReceiverTypeData::count_offset()); + int base_receiver_offset = in_bytes(ReceiverTypeData::receiver_offset(0)); + int receiver_to_count_step = in_bytes(ReceiverTypeData::receiver_count_offset(0)) - base_receiver_offset; + + // Adjust for MDP offsets. + poly_count_offset += mdp_offset; + +#ifdef ASSERT + int real_poly_count_offset = mdp_offset + in_bytes(ReceiverTypeData::count_offset()); + assert(poly_count_offset == real_poly_count_offset, "poly counter math"); +#endif + + // Corner case: no profile table. Increment poly counter and exit. + if (ReceiverTypeData::row_limit() == 0) { + increment(Address(mdp, poly_count_offset), DataLayout::counter_increment); + return; + } + + Label L_found_recv; + profile_receiver_type_helper(recv, mdp, L_found_recv, mdp_offset, ReceiverTypeData::base_of_megamorphic_type_data(), ReceiverTypeData::row_limit()); + // Increment polymorphic counter instead of receiver slot. + Register offset = rscratch2; + mov(offset, poly_count_offset); + Label L_count_update; + b(L_count_update); + // Found a receiver, convert its slot offset to corresponding count offset. bind(L_found_recv); add(offset, offset, receiver_to_count_step); - // Finally, update the counter bind(L_count_update); increment(Address(mdp, offset), DataLayout::counter_increment); } +void MacroAssembler::profile_array_type_at_load(Register recv, Register mdp, int mdp_offset) { + int base_receiver_offset = in_bytes(ArrayLoadData::receiver_offset(0)); + int receiver_to_count_step = in_bytes(ArrayLoadData::receiver_count_offset(0)) - base_receiver_offset; + int flat_nullable_count_offset = in_bytes(ArrayLoadData::flat_nullable_count_offset()); + int flat_nullfree_atomic_count_offset = in_bytes(ArrayLoadData::flat_nullfree_atomic_count_offset()); + int flat_nullfree_not_atomic_count_offset = in_bytes(ArrayLoadData::flat_nullfree_not_atomic_count_offset()); + + // Adjust for MDP offsets. Slots are pointer-sized, so is the global offset. + flat_nullable_count_offset += mdp_offset; + flat_nullfree_atomic_count_offset += mdp_offset; + flat_nullfree_not_atomic_count_offset += mdp_offset; + +#ifdef ASSERT + int real_flat_nullable_count_offset = mdp_offset + in_bytes(ArrayLoadData::flat_nullable_count_offset()); + assert(flat_nullable_count_offset == real_flat_nullable_count_offset, "poly counter math"); + int real_flat_nullfree_atomic_count_offset = mdp_offset + in_bytes(ArrayLoadData::flat_nullfree_atomic_count_offset()); + assert(flat_nullfree_atomic_count_offset == real_flat_nullfree_atomic_count_offset, "poly counter math"); + int real_flat_nullfree_not_atomic_count_offset = mdp_offset + in_bytes(ArrayLoadData::flat_nullfree_not_atomic_count_offset()); + assert(flat_nullfree_not_atomic_count_offset == real_flat_nullfree_not_atomic_count_offset, "poly counter math"); +#endif + + Label L_found_recv; + // Corner case: no profile table. Increment poly counter and exit. + if (ReceiverTypeData::row_limit() != 0) { + profile_receiver_type_helper(recv, mdp, L_found_recv, mdp_offset, ArrayLoadData::base_of_megamorphic_type_data(), ArrayLoadData::row_limit()); + } + + Register offset = rscratch2; + int layout_kind_offset = in_bytes(FlatArrayKlass::layout_kind_offset()); + Label null_free_non_atomic, null_free_atomic, nullable_atomic_flat, failure, L_count_update; + ldrw(rscratch1, Address(recv, layout_kind_offset)); + cmpw(rscratch1, (int)LayoutKind::NULL_FREE_ATOMIC_FLAT); + br(NE, null_free_non_atomic); + mov(offset, flat_nullfree_atomic_count_offset); + + b(L_count_update); + bind(null_free_non_atomic); + ldrw(rscratch1, Address(recv, layout_kind_offset)); + cmpw(rscratch1, (int)LayoutKind::NULL_FREE_NON_ATOMIC_FLAT); + br(NE, nullable_atomic_flat); + mov(offset, flat_nullfree_not_atomic_count_offset); + + b(L_count_update); + bind(nullable_atomic_flat); + ldrw(rscratch1, Address(recv, layout_kind_offset)); + cmpw(rscratch1, (int)LayoutKind::NULLABLE_ATOMIC_FLAT); + br(NE, failure); + mov(offset, flat_nullable_count_offset); + + b(L_count_update); + bind(failure); + stop("unexpected flat array"); + + // Found a receiver, convert its slot offset to corresponding count offset. + bind(L_found_recv); + add(offset, offset, receiver_to_count_step); + + bind(L_count_update); + increment(Address(mdp, offset), DataLayout::counter_increment); +} void MacroAssembler::call_VM_leaf_base(address entry_point, int number_of_arguments, diff --git a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp index 12e54b77a67..2295d128395 100644 --- a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp @@ -1177,7 +1177,11 @@ class MacroAssembler: public Assembler { Address argument_address(RegisterOrConstant arg_slot, int extra_slot_offset = 0); + void profile_receiver_type_helper(Register recv, Register mdp, + Label &L_found_recv, int mdp_offset, int base, uint row_limit); + void profile_receiver_type(Register recv, Register mdp, int mdp_offset); + void profile_array_type_at_load(Register recv, Register mdp, int mdp_offset); void verify_sve_vector_length(Register tmp = rscratch1); void reinitialize_ptrue() { diff --git a/src/hotspot/cpu/aarch64/templateTable_aarch64.cpp b/src/hotspot/cpu/aarch64/templateTable_aarch64.cpp index 698ca7fd67a..e40170ba533 100644 --- a/src/hotspot/cpu/aarch64/templateTable_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/templateTable_aarch64.cpp @@ -818,7 +818,7 @@ void TemplateTable::aaload() // r0: array // r1: index index_check(r0, r1); // leaves index in r1, kills rscratch1 - __ profile_array_type(r2, r0, r4); + __ profile_multiple_array_types(r2, r0, r4); if (UseArrayFlattening) { Label is_flat_array, done; @@ -1130,8 +1130,8 @@ void TemplateTable::aastore() { index_check(r3, r2); // kills r1 - __ profile_array_type(r4, r3, r5); - __ profile_multiple_element_types(r4, r0, r5, r6); + __ profile_array_type(r4, r3, r5); + __ profile_multiple_element_types(r4, r0, r5); __ add(r4, r2, arrayOopDesc::base_offset_in_bytes(T_OBJECT) >> LogBytesPerHeapOop); Address element_address(r3, r4, Address::uxtw(LogBytesPerHeapOop)); diff --git a/src/hotspot/cpu/arm/c1_LIRAssembler_arm.cpp b/src/hotspot/cpu/arm/c1_LIRAssembler_arm.cpp index 62424d9df22..1c751692ace 100644 --- a/src/hotspot/cpu/arm/c1_LIRAssembler_arm.cpp +++ b/src/hotspot/cpu/arm/c1_LIRAssembler_arm.cpp @@ -2559,6 +2559,10 @@ void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) { fatal("Type profiling not implemented on this platform"); } +void LIR_Assembler::emit_profile_multiple_array_types(LIR_OpProfileMultipleArrayTypes* op) { + Unimplemented(); +} + void LIR_Assembler::emit_profile_inline_type(LIR_OpProfileInlineType* op) { Unimplemented(); } diff --git a/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp b/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp index cd93b9c882b..be8fd16173d 100644 --- a/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp +++ b/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp @@ -3021,6 +3021,10 @@ void LIR_Assembler::emit_profile_inline_type(LIR_OpProfileInlineType* op) { Unimplemented(); } +void LIR_Assembler::emit_profile_multiple_array_types(LIR_OpProfileMultipleArrayTypes* op) { + Unimplemented(); +} + void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) { assert(op->crc()->is_single_cpu(), "crc must be register"); assert(op->val()->is_single_cpu(), "byte value must be register"); diff --git a/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp b/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp index a5359f81ea7..377bb7e185a 100644 --- a/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp +++ b/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp @@ -1885,6 +1885,14 @@ void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) { COMMENT("} emit_profile_type"); } +void LIR_Assembler::emit_profile_inline_type(LIR_OpProfileInlineType* op) { + Unimplemented(); +} + +void LIR_Assembler::emit_profile_multiple_array_types(LIR_OpProfileMultipleArrayTypes* op) { + Unimplemented(); +} + void LIR_Assembler::align_backward_branch_target() { } void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) { diff --git a/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp b/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp index 7808771119c..be2d1e1fcb4 100644 --- a/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp +++ b/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp @@ -3072,6 +3072,10 @@ void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) { } } +void LIR_Assembler::emit_profile_multiple_array_types(LIR_OpProfileMultipleArrayTypes* op) { + Unimplemented(); +} + void LIR_Assembler::emit_profile_inline_type(LIR_OpProfileInlineType* op) { Unimplemented(); } diff --git a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp index 0b219f53817..db72060c54c 100644 --- a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp @@ -3175,7 +3175,47 @@ void LIR_Assembler::emit_profile_inline_type(LIR_OpProfileInlineType* op) { __ bind(not_inline_type); } +void LIR_Assembler::emit_profile_multiple_array_types(LIR_OpProfileMultipleArrayTypes* op) { + Register array = op->array()->as_pointer_register(); + Register tmp1 = op->tmp1()->as_pointer_register(); + Register tmp2 = op->tmp2()->as_pointer_register(); + ciMethodData* md = op->md(); + Label not_flat, done; + __ test_non_flat_array_oop(array, tmp1, not_flat); + + Register klass = tmp1; + __ load_klass(klass, array, tmp2); + + Register mdo = tmp2; + __ mov_metadata(mdo, md->constant_encoding()); + + int mdp_offset = md->byte_offset_of_slot(op->load(), in_ByteSize(0)); + __ profile_array_type_at_load(tmp1, mdo, mdp_offset); + + __ jmp(done); + __ bind(not_flat); + __ mov_metadata(mdo, md->constant_encoding()); + + Label null_free; + + __ test_null_free_array_oop(array, tmp1, null_free); + + { + Address counter_addr(mdo, md->byte_offset_of_slot(op->load(), ArrayLoadData::not_flat_nullable_count_offset())); + __ addptr(counter_addr, DataLayout::counter_increment); + } + + __ jmp(done); + __ bind(null_free); + + { + Address counter_addr(mdo, md->byte_offset_of_slot(op->load(), ArrayLoadData::not_flat_null_free_count_offset())); + __ addptr(counter_addr, DataLayout::counter_increment); + } + + __ bind(done); +} void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) { __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no)); } diff --git a/src/hotspot/cpu/x86/interp_masm_x86.cpp b/src/hotspot/cpu/x86/interp_masm_x86.cpp index d5e7b22b16a..b8b2e780f83 100644 --- a/src/hotspot/cpu/x86/interp_masm_x86.cpp +++ b/src/hotspot/cpu/x86/interp_masm_x86.cpp @@ -1614,9 +1614,9 @@ void InterpreterMacroAssembler::profile_switch_case(Register index, } } -template void InterpreterMacroAssembler::profile_array_type(Register mdp, - Register array, - Register tmp) { +void InterpreterMacroAssembler::profile_array_type(Register mdp, + Register array, + Register tmp) { if (ProfileInterpreter) { Label profile_continue; @@ -1624,19 +1624,19 @@ template void InterpreterMacroAssembler::profile_array_type(Re test_method_data_pointer(mdp, profile_continue); mov(tmp, array); - profile_obj_type(tmp, Address(mdp, in_bytes(ArrayData::array_offset()))); + profile_obj_type(tmp, Address(mdp, in_bytes(ArrayStoreData::array_offset()))); Label not_flat; test_non_flat_array_oop(array, tmp, not_flat); - set_mdp_flag_at(mdp, ArrayData::flat_array_byte_constant()); + set_mdp_flag_at(mdp, ArrayStoreData::flat_array_byte_constant()); bind(not_flat); Label not_null_free; test_non_null_free_array_oop(array, tmp, not_null_free); - set_mdp_flag_at(mdp, ArrayData::null_free_array_byte_constant()); + set_mdp_flag_at(mdp, ArrayStoreData::null_free_array_byte_constant()); bind(not_null_free); @@ -1644,15 +1644,40 @@ template void InterpreterMacroAssembler::profile_array_type(Re } } -template void InterpreterMacroAssembler::profile_array_type(Register mdp, - Register array, - Register tmp); -template void InterpreterMacroAssembler::profile_array_type(Register mdp, - Register array, - Register tmp); +void InterpreterMacroAssembler::profile_multiple_array_types(Register mdp, + Register array, + Register tmp) { + if (ProfileInterpreter) { + Label profile_continue; + + // If no method data exists, go to profile_continue. + test_method_data_pointer(mdp, profile_continue); + + Label not_flat; + test_non_flat_array_oop(array, tmp, not_flat); + + load_klass(tmp, array, rscratch1); + profile_array_type_at_load(tmp, mdp, 0); + + jmp(profile_continue); + bind(not_flat); + Label not_null_free; + test_non_null_free_array_oop(array, tmp, not_null_free); + + increment_mdp_data_at(mdp, in_bytes(ArrayLoadData::not_flat_null_free_count_offset())); + + jmp(profile_continue); + + bind(not_null_free); + + increment_mdp_data_at(mdp, in_bytes(ArrayLoadData::not_flat_nullable_count_offset())); + + bind(profile_continue); + } +} -void InterpreterMacroAssembler::profile_multiple_element_types(Register mdp, Register element, Register tmp, const Register tmp2) { +void InterpreterMacroAssembler::profile_multiple_element_types(Register mdp, Register element, Register tmp) { if (ProfileInterpreter) { Label profile_continue; diff --git a/src/hotspot/cpu/x86/interp_masm_x86.hpp b/src/hotspot/cpu/x86/interp_masm_x86.hpp index ed4b8003ed4..63b719badaf 100644 --- a/src/hotspot/cpu/x86/interp_masm_x86.hpp +++ b/src/hotspot/cpu/x86/interp_masm_x86.hpp @@ -260,9 +260,10 @@ class InterpreterMacroAssembler: public MacroAssembler { void profile_switch_default(Register mdp); void profile_switch_case(Register index_in_scratch, Register mdp, Register scratch2); - template void profile_array_type(Register mdp, Register array, Register tmp); + void profile_array_type(Register mdp, Register array, Register tmp); + void profile_multiple_array_types(Register mdp, Register array, Register tmp); - void profile_multiple_element_types(Register mdp, Register element, Register tmp, const Register tmp2); + void profile_multiple_element_types(Register mdp, Register element, Register tmp); void profile_element_type(Register mdp, Register element, Register tmp); void profile_acmp(Register mdp, Register left, Register right, Register tmp); diff --git a/src/hotspot/cpu/x86/macroAssembler_x86.cpp b/src/hotspot/cpu/x86/macroAssembler_x86.cpp index 610b8cf0051..5d8d57fe060 100644 --- a/src/hotspot/cpu/x86/macroAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/macroAssembler_x86.cpp @@ -4955,39 +4955,22 @@ Address MacroAssembler::argument_address(RegisterOrConstant arg_slot, return Address(rsp, scale_reg, scale_factor, offset); } -// Handle the receiver type profile update given the "recv" klass. -// -// Normally updates the ReceiverData (RD) that starts at "mdp" + "mdp_offset". -// If there are no matching or claimable receiver entries in RD, updates -// the polymorphic counter. -// -// This code expected to run by either the interpreter or JIT-ed code, without -// extra synchronization. For safety, receiver cells are claimed atomically, which -// avoids grossly misrepresenting the profiles under concurrent updates. For speed, -// counter updates are not atomic. -// -void MacroAssembler::profile_receiver_type(Register recv, Register mdp, int mdp_offset) { - int base_receiver_offset = in_bytes(ReceiverTypeData::receiver_offset(0)); - int end_receiver_offset = in_bytes(ReceiverTypeData::receiver_offset(ReceiverTypeData::row_limit())); - int poly_count_offset = in_bytes(CounterData::count_offset()); - int receiver_step = in_bytes(ReceiverTypeData::receiver_offset(1)) - base_receiver_offset; - int receiver_to_count_step = in_bytes(ReceiverTypeData::receiver_count_offset(0)) - base_receiver_offset; +void MacroAssembler::profile_receiver_type_helper(Register recv, Register mdp, Label &L_found_recv, int mdp_offset, int base, uint row_limit) { + int base_receiver_offset = in_bytes(MegamorphicTypeData::receiver_offset(base, 0)); + int end_receiver_offset = in_bytes(MegamorphicTypeData::receiver_offset(base, row_limit)); + int receiver_step = in_bytes(MegamorphicTypeData::receiver_offset(base, 1)) - base_receiver_offset; + int receiver_to_count_step = in_bytes(MegamorphicTypeData::receiver_count_offset(base, 0)) - base_receiver_offset; - // Adjust for MDP offsets. Slots are pointer-sized, so is the global offset. assert(is_aligned(mdp_offset, BytesPerWord), "sanity"); base_receiver_offset += mdp_offset; end_receiver_offset += mdp_offset; - poly_count_offset += mdp_offset; - // Scale down to optimize encoding. Slots are pointer-sized. assert(is_aligned(base_receiver_offset, BytesPerWord), "sanity"); assert(is_aligned(end_receiver_offset, BytesPerWord), "sanity"); - assert(is_aligned(poly_count_offset, BytesPerWord), "sanity"); assert(is_aligned(receiver_step, BytesPerWord), "sanity"); assert(is_aligned(receiver_to_count_step, BytesPerWord), "sanity"); base_receiver_offset >>= LogBytesPerWord; end_receiver_offset >>= LogBytesPerWord; - poly_count_offset >>= LogBytesPerWord; receiver_step >>= LogBytesPerWord; receiver_to_count_step >>= LogBytesPerWord; @@ -4995,27 +4978,19 @@ void MacroAssembler::profile_receiver_type(Register recv, Register mdp, int mdp_ // We are about to walk the MDO slots without asking for offsets. // Check that our math hits all the right spots. for (uint c = 0; c < ReceiverTypeData::row_limit(); c++) { - int real_recv_offset = mdp_offset + in_bytes(ReceiverTypeData::receiver_offset(c)); - int real_count_offset = mdp_offset + in_bytes(ReceiverTypeData::receiver_count_offset(c)); + int real_recv_offset = mdp_offset + in_bytes(MegamorphicTypeData::receiver_offset(base, c)); + int real_count_offset = mdp_offset + in_bytes(MegamorphicTypeData::receiver_count_offset(base, c)); int offset = base_receiver_offset + receiver_step*c; int count_offset = offset + receiver_to_count_step; assert((offset << LogBytesPerWord) == real_recv_offset, "receiver slot math"); assert((count_offset << LogBytesPerWord) == real_count_offset, "receiver count math"); } - int real_poly_count_offset = mdp_offset + in_bytes(CounterData::count_offset()); - assert(poly_count_offset << LogBytesPerWord == real_poly_count_offset, "poly counter math"); #endif - // Corner case: no profile table. Increment poly counter and exit. - if (ReceiverTypeData::row_limit() == 0) { - addptr(Address(mdp, poly_count_offset, Address::times_ptr), DataLayout::counter_increment); - return; - } - Register offset = rscratch1; Label L_loop_search_receiver, L_loop_search_empty; - Label L_restart, L_found_recv, L_found_empty, L_count_update; + Label L_restart, L_found_empty, L_polymorphic; // The code here recognizes three major cases: // A. Fastest: receiver found in the table @@ -5045,20 +5020,21 @@ void MacroAssembler::profile_receiver_type(Register recv, Register mdp, int mdp_ // if (receiver(i) == recv) goto found_recv(i); // } // - // // Fast: no receiver, but profile is not full + // // Fast: no receiver, but profile is full // for (i = 0; i < receiver_count(); i++) { // if (receiver(i) == null) goto found_null(i); // } - // - // // Slow: profile is full, polymorphic case - // count++; - // return + // goto polymorphic // // // Slow: try to install receiver // found_null(i): // CAS(&receiver(i), null, recv); // goto restart // + // polymorphic: + // count++; + // return + // // found_recv(i): // *receiver_count(i)++ // @@ -5068,27 +5044,23 @@ void MacroAssembler::profile_receiver_type(Register recv, Register mdp, int mdp_ // Fastest: receiver is already installed movptr(offset, base_receiver_offset); bind(L_loop_search_receiver); - cmpptr(recv, Address(mdp, offset, Address::times_ptr)); - jccb(Assembler::equal, L_found_recv); + cmpptr(recv, Address(mdp, offset, Address::times_ptr)); + jcc(Assembler::equal, L_found_recv); addptr(offset, receiver_step); cmpptr(offset, end_receiver_offset); jccb(Assembler::notEqual, L_loop_search_receiver); - // Fast: no receiver, but profile is not full + // Fast: no receiver, but profile is full movptr(offset, base_receiver_offset); bind(L_loop_search_empty); - cmpptr(Address(mdp, offset, Address::times_ptr), NULL_WORD); - jccb(Assembler::equal, L_found_empty); + cmpptr(Address(mdp, offset, Address::times_ptr), NULL_WORD); + jccb(Assembler::equal, L_found_empty); addptr(offset, receiver_step); cmpptr(offset, end_receiver_offset); jccb(Assembler::notEqual, L_loop_search_empty); + jmpb(L_polymorphic); - // Slow: Receiver is not found and table is full. - // Increment polymorphic counter instead of receiver slot. - movptr(offset, poly_count_offset); - jmpb(L_count_update); - - // Slowest: try to install receiver + // Slow: try to install receiver bind(L_found_empty); // Atomically swing receiver slot: null -> recv. @@ -5103,8 +5075,8 @@ void MacroAssembler::profile_receiver_type(Register recv, Register mdp, int mdp_ Register shifted_recv = recv; if (recv == rax || mdp == rax) { spare_reg = (recv != rbx && mdp != rbx) ? rbx : - (recv != rcx && mdp != rcx) ? rcx : - rdx; + (recv != rcx && mdp != rcx) ? rcx : + rdx; assert_different_registers(mdp, recv, offset, spare_reg); push(spare_reg); @@ -5140,11 +5112,130 @@ void MacroAssembler::profile_receiver_type(Register recv, Register mdp, int mdp_ // and just restart the search from the beginning. jmpb(L_restart); + // Counter updates: + + // Increment polymorphic counter instead of receiver slot. + bind(L_polymorphic); +} + +// Handle the receiver type profile update given the "recv" klass. +// +// Normally updates the ReceiverData (RD) that starts at "mdp" + "mdp_offset". +// If there are no matching or claimable receiver entries in RD, updates +// the polymorphic counter. +// +// This code expected to run by either the interpreter or JIT-ed code, without +// extra synchronization. For safety, receiver cells are claimed atomically, which +// avoids grossly misrepresenting the profiles under concurrent updates. For speed, +// counter updates are not atomic. +// +void MacroAssembler::profile_receiver_type(Register recv, Register mdp, int mdp_offset) { + int poly_count_offset = in_bytes(ReceiverTypeData::count_offset()); + int base_receiver_offset = in_bytes(ReceiverTypeData::receiver_offset(0)); + int receiver_to_count_step = in_bytes(ReceiverTypeData::receiver_count_offset(0)) - base_receiver_offset; + + // Adjust for MDP offsets. Slots are pointer-sized, so is the global offset. + assert(is_aligned(mdp_offset, BytesPerWord), "sanity"); + poly_count_offset += mdp_offset; + + // Scale down to optimize encoding. Slots are pointer-sized. + assert(is_aligned(poly_count_offset, BytesPerWord), "sanity"); + assert(is_aligned(receiver_to_count_step, BytesPerWord), "sanity"); + poly_count_offset >>= LogBytesPerWord; + receiver_to_count_step >>= LogBytesPerWord; + +#ifdef ASSERT + int real_poly_count_offset = mdp_offset + in_bytes(ReceiverTypeData::count_offset()); + assert(poly_count_offset << LogBytesPerWord == real_poly_count_offset, "poly counter math"); +#endif + + // Corner case: no profile table. Increment poly counter and exit. + if (ReceiverTypeData::row_limit() == 0) { + addptr(Address(mdp, poly_count_offset, Address::times_ptr), DataLayout::counter_increment); + return; + } + + Label L_found_recv; + profile_receiver_type_helper(recv, mdp, L_found_recv, mdp_offset, ReceiverTypeData::base_of_megamorphic_type_data(), ReceiverTypeData::row_limit()); + Register offset = rscratch1; + movptr(offset, poly_count_offset); + Label L_count_update; + jmpb(L_count_update); + // Found a receiver, convert its slot offset to corresponding count offset. bind(L_found_recv); addptr(offset, receiver_to_count_step); - // Finally, update the counter + bind(L_count_update); + addptr(Address(mdp, offset, Address::times_ptr), DataLayout::counter_increment); +} + +void MacroAssembler::profile_array_type_at_load(Register recv, Register mdp, int mdp_offset) { + int base_receiver_offset = in_bytes(ArrayLoadData::receiver_offset(0)); + int receiver_to_count_step = in_bytes(ArrayLoadData::receiver_count_offset(0)) - base_receiver_offset; + int flat_nullable_count_offset = in_bytes(ArrayLoadData::flat_nullable_count_offset()); + int flat_nullfree_atomic_count_offset = in_bytes(ArrayLoadData::flat_nullfree_atomic_count_offset()); + int flat_nullfree_not_atomic_count_offset = in_bytes(ArrayLoadData::flat_nullfree_not_atomic_count_offset()); + + // Adjust for MDP offsets. Slots are pointer-sized, so is the global offset. + assert(is_aligned(mdp_offset, BytesPerWord), "sanity"); + flat_nullable_count_offset += mdp_offset; + flat_nullfree_atomic_count_offset += mdp_offset; + flat_nullfree_not_atomic_count_offset += mdp_offset; + + // Scale down to optimize encoding. Slots are pointer-sized. + assert(is_aligned(flat_nullable_count_offset, BytesPerWord), "sanity"); + assert(is_aligned(flat_nullfree_atomic_count_offset, BytesPerWord), "sanity"); + assert(is_aligned(flat_nullfree_not_atomic_count_offset, BytesPerWord), "sanity"); + assert(is_aligned(receiver_to_count_step, BytesPerWord), "sanity"); + flat_nullable_count_offset >>= LogBytesPerWord; + flat_nullfree_atomic_count_offset >>= LogBytesPerWord; + flat_nullfree_not_atomic_count_offset >>= LogBytesPerWord; + receiver_to_count_step >>= LogBytesPerWord; + +#ifdef ASSERT + int real_flat_nullable_count_offset = mdp_offset + in_bytes(ArrayLoadData::flat_nullable_count_offset()); + assert(flat_nullable_count_offset << LogBytesPerWord == real_flat_nullable_count_offset, "poly counter math"); + int real_flat_nullfree_atomic_count_offset = mdp_offset + in_bytes(ArrayLoadData::flat_nullfree_atomic_count_offset()); + assert(flat_nullfree_atomic_count_offset << LogBytesPerWord == real_flat_nullfree_atomic_count_offset, "poly counter math"); + int real_flat_nullfree_not_atomic_count_offset = mdp_offset + in_bytes(ArrayLoadData::flat_nullfree_not_atomic_count_offset()); + assert(flat_nullfree_not_atomic_count_offset << LogBytesPerWord == real_flat_nullfree_not_atomic_count_offset, "poly counter math"); +#endif + + Label L_found_recv; + // Corner case: no profile table. Increment poly counter and exit. + if (ReceiverTypeData::row_limit() != 0) { + profile_receiver_type_helper(recv, mdp, L_found_recv, mdp_offset, ArrayLoadData::base_of_megamorphic_type_data(), ArrayLoadData::row_limit()); + } + Register offset = rscratch1; + int layout_kind_offset = in_bytes(FlatArrayKlass::layout_kind_offset()); + Label null_free_non_atomic, null_free_atomic, nullable_atomic_flat, failure, L_count_update; + cmpl(Address(recv, layout_kind_offset), (int)LayoutKind::NULL_FREE_ATOMIC_FLAT); + jccb(Assembler::notEqual, null_free_non_atomic); + movptr(offset, flat_nullfree_atomic_count_offset); + + jmp(L_count_update); + bind(null_free_non_atomic); + cmpl(Address(recv, layout_kind_offset), (int)LayoutKind::NULL_FREE_NON_ATOMIC_FLAT); + jccb(Assembler::notEqual, nullable_atomic_flat); + movptr(offset, flat_nullfree_not_atomic_count_offset); + + jmp(L_count_update); + bind(nullable_atomic_flat); + cmpl(Address(recv, layout_kind_offset), (int)LayoutKind::NULLABLE_ATOMIC_FLAT); + jccb(Assembler::notEqual, failure); + movptr(offset, flat_nullable_count_offset); + + jmpb(L_count_update); + bind(failure); + stop("unexpected flat array"); + + if (ReceiverTypeData::row_limit() != 0) { + // Found a receiver, convert its slot offset to corresponding count offset. + bind(L_found_recv); + addptr(offset, receiver_to_count_step); + } + bind(L_count_update); addptr(Address(mdp, offset, Address::times_ptr), DataLayout::counter_increment); } diff --git a/src/hotspot/cpu/x86/macroAssembler_x86.hpp b/src/hotspot/cpu/x86/macroAssembler_x86.hpp index b9b31d21391..dffdbbcaa9a 100644 --- a/src/hotspot/cpu/x86/macroAssembler_x86.hpp +++ b/src/hotspot/cpu/x86/macroAssembler_x86.hpp @@ -708,7 +708,11 @@ class MacroAssembler: public Assembler { // method handles (JSR 292) Address argument_address(RegisterOrConstant arg_slot, int extra_slot_offset = 0); + void profile_receiver_type_helper(Register recv, Register mdp, + Label &L_found_recv, int mdp_offset, int base, uint row_limit); + void profile_receiver_type(Register recv, Register mdp, int mdp_offset); + void profile_array_type_at_load(Register recv, Register mdp, int mdp_offset); // Debugging diff --git a/src/hotspot/cpu/x86/templateTable_x86.cpp b/src/hotspot/cpu/x86/templateTable_x86.cpp index 4c1ccde0e52..e5af853af23 100644 --- a/src/hotspot/cpu/x86/templateTable_x86.cpp +++ b/src/hotspot/cpu/x86/templateTable_x86.cpp @@ -782,7 +782,7 @@ void TemplateTable::aaload() { Register index = rax; index_check(array, index); // kills rbx - __ profile_array_type(rbx, array, rcx); + __ profile_multiple_array_types(rbx, array, rcx); if (UseArrayFlattening) { Label is_flat_array, done; __ test_flat_array_oop(array, rbx, is_flat_array); @@ -1092,8 +1092,8 @@ void TemplateTable::aastore() { index_check_without_pop(rdx, rcx); // kills rbx - __ profile_array_type(rdi, rdx, rbx); - __ profile_multiple_element_types(rdi, rax, rbx, rcx); + __ profile_array_type(rdi, rdx, rbx); + __ profile_multiple_element_types(rdi, rax, rbx); __ testptr(rax, rax); __ jcc(Assembler::zero, is_null); diff --git a/src/hotspot/share/c1/c1_LIR.cpp b/src/hotspot/share/c1/c1_LIR.cpp index 65b28be7911..a5a612f190c 100644 --- a/src/hotspot/share/c1/c1_LIR.cpp +++ b/src/hotspot/share/c1/c1_LIR.cpp @@ -986,6 +986,16 @@ void LIR_OpVisitState::visit(LIR_Op* op) { do_temp(opProfileInlineType->_tmp); break; } + + case lir_profile_multiple_array_types: { + assert(op->as_OpProfileMultipleArrayTypes() != nullptr, "must be"); + LIR_OpProfileMultipleArrayTypes* opProfileMultipleArrayTypes = (LIR_OpProfileMultipleArrayTypes*)op; + do_input(opProfileMultipleArrayTypes->_array); do_temp(opProfileMultipleArrayTypes->_array); + do_temp(opProfileMultipleArrayTypes->_tmp1); + do_temp(opProfileMultipleArrayTypes->_tmp2); + + break; + } default: op->visit(this); } @@ -1209,6 +1219,10 @@ void LIR_OpProfileInlineType::emit_code(LIR_Assembler* masm) { masm->emit_profile_inline_type(this); } +void LIR_OpProfileMultipleArrayTypes::emit_code(LIR_Assembler* masm) { + masm->emit_profile_multiple_array_types(this); +} + // LIR_List LIR_List::LIR_List(Compilation* compilation, BlockBegin* block) : _operations(8) @@ -1930,6 +1944,7 @@ const char * LIR_Op::name() const { case lir_profile_type: s = "profile_type"; break; // LIR_OpProfileInlineType case lir_profile_inline_type: s = "profile_inline_type"; break; + case lir_profile_multiple_array_types: s = "profile_multiple_array_types"; break; // LIR_OpAssert #ifdef ASSERT case lir_assert: s = "assert"; break; @@ -2268,6 +2283,9 @@ void LIR_OpProfileInlineType::print_instr(outputStream* out) const { tmp()->print(out); out->print(" "); } +void LIR_OpProfileMultipleArrayTypes::print_instr(outputStream* out) const { +} + #endif // PRODUCT // Implementation of LIR_InsertionBuffer diff --git a/src/hotspot/share/c1/c1_LIR.hpp b/src/hotspot/share/c1/c1_LIR.hpp index 91a80bfdd12..371d6f5dd42 100644 --- a/src/hotspot/share/c1/c1_LIR.hpp +++ b/src/hotspot/share/c1/c1_LIR.hpp @@ -897,6 +897,7 @@ class LIR_OpLoadKlass; class LIR_OpProfileCall; class LIR_OpProfileType; class LIR_OpProfileInlineType; +class LIR_OpProfileMultipleArrayTypes; #ifdef ASSERT class LIR_OpAssert; #endif @@ -1012,6 +1013,7 @@ enum LIR_Code { , lir_profile_call , lir_profile_type , lir_profile_inline_type + , lir_profile_multiple_array_types , end_opMDOProfile , begin_opAssert , lir_assert @@ -1158,6 +1160,7 @@ class LIR_Op: public CompilationResourceObj { virtual LIR_OpProfileCall* as_OpProfileCall() { return nullptr; } virtual LIR_OpProfileType* as_OpProfileType() { return nullptr; } virtual LIR_OpProfileInlineType* as_OpProfileInlineType() { return nullptr; } + virtual LIR_OpProfileMultipleArrayTypes* as_OpProfileMultipleArrayTypes() { return nullptr; } #ifdef ASSERT virtual LIR_OpAssert* as_OpAssert() { return nullptr; } #endif @@ -2146,6 +2149,34 @@ class LIR_OpProfileInlineType : public LIR_Op { virtual void print_instr(outputStream* out) const PRODUCT_RETURN; }; +class LIR_OpProfileMultipleArrayTypes : public LIR_Op { + friend class LIR_OpVisitState; + ciMethodData* _md; + ciArrayLoadData* _load; + LIR_Opr _array; + LIR_Opr _tmp1; + LIR_Opr _tmp2; +public: + LIR_OpProfileMultipleArrayTypes(ciMethodData* md, ciArrayLoadData* load, LIR_Opr array, LIR_Opr tmp1, LIR_Opr tmp2) + : LIR_Op(lir_profile_multiple_array_types, LIR_OprFact::illegalOpr, nullptr) + , _md(md) + , _load(load) + , _array(array) + , _tmp1(tmp1) + , _tmp2(tmp2) { + + } + ciMethodData* md() const { return _md; } + ciArrayLoadData* load() const { return _load; } + LIR_Opr array() const { return _array; } + LIR_Opr tmp1() const { return _tmp1; } + LIR_Opr tmp2() const { return _tmp2; } + + virtual void emit_code(LIR_Assembler* masm); + virtual LIR_OpProfileMultipleArrayTypes* as_OpProfileMultipleArrayTypes() { return this; } + virtual void print_instr(outputStream* out) const PRODUCT_RETURN; +}; + class LIR_InsertionBuffer; //--------------------------------LIR_List--------------------------------------------------- @@ -2447,6 +2478,9 @@ class LIR_List: public CompilationResourceObj { void profile_inline_type(LIR_Address* mdp, LIR_Opr obj, int flag, LIR_Opr tmp, bool not_null) { append(new LIR_OpProfileInlineType(LIR_OprFact::address(mdp), obj, flag, tmp, not_null)); } + void profile_multiple_array_types(ciMethodData* md, ciArrayLoadData* load, LIR_Opr array, LIR_Opr tmp1, LIR_Opr tmp2) { + append(new LIR_OpProfileMultipleArrayTypes(md, load, array, tmp1, tmp2)); + } void xadd(LIR_Opr src, LIR_Opr add, LIR_Opr res, LIR_Opr tmp) { append(new LIR_Op2(lir_xadd, src, add, res, tmp)); } void xchg(LIR_Opr src, LIR_Opr set, LIR_Opr res, LIR_Opr tmp) { append(new LIR_Op2(lir_xchg, src, set, res, tmp)); } diff --git a/src/hotspot/share/c1/c1_LIRAssembler.hpp b/src/hotspot/share/c1/c1_LIRAssembler.hpp index 6b98a0b97dd..f68dcee3bd6 100644 --- a/src/hotspot/share/c1/c1_LIRAssembler.hpp +++ b/src/hotspot/share/c1/c1_LIRAssembler.hpp @@ -213,6 +213,7 @@ class LIR_Assembler: public CompilationResourceObj { void emit_profile_call(LIR_OpProfileCall* op); void emit_profile_type(LIR_OpProfileType* op); void emit_profile_inline_type(LIR_OpProfileInlineType* op); + void emit_profile_multiple_array_types(LIR_OpProfileMultipleArrayTypes* op); void emit_std_entries(); void emit_std_entry(CodeOffsets::Entries entry, const CompiledEntrySignature* ces); void add_scalarized_debug_info(int call_offset); diff --git a/src/hotspot/share/c1/c1_LIRGenerator.cpp b/src/hotspot/share/c1/c1_LIRGenerator.cpp index 0227615241c..41269ac73f2 100644 --- a/src/hotspot/share/c1/c1_LIRGenerator.cpp +++ b/src/hotspot/share/c1/c1_LIRGenerator.cpp @@ -1973,7 +1973,7 @@ void LIRGenerator::do_StoreIndexed(StoreIndexed* x) { profile_array_type(x, md, store_data); assert(store_data->is_ArrayStoreData(), "incorrect profiling entry"); if (x->array()->maybe_null_free_array()) { - profile_null_free_array(array, md, data); + profile_null_free_array(array, md, store_data); } } } @@ -2349,7 +2349,7 @@ void LIRGenerator::do_LoadIndexed(LoadIndexed* x) { data = md->bci_to_data(bci); assert(data != nullptr && data->is_ArrayLoadData(), "incorrect profiling entry"); ciArrayLoadData* load_data = (ciArrayLoadData*)data; - profile_array_type(x, md, load_data); + profile_multiple_array_types(x, array, md, load_data); } } @@ -2381,10 +2381,6 @@ void LIRGenerator::do_LoadIndexed(LoadIndexed* x) { LIR_Opr result = rlock_result(x, x->elt_type()); LoadFlattenedArrayStub* slow_path = nullptr; - if (x->should_profile() && x->array()->maybe_null_free_array()) { - profile_null_free_array(array, md, data); - } - if (x->elt_type() == T_OBJECT && x->array()->maybe_flat_array()) { assert(x->delayed() == nullptr, "Delayed LoadIndexed only apply to loaded_flat_arrays"); index.load_item(); @@ -3010,9 +3006,8 @@ void LIRGenerator::profile_flags(ciMethodData* md, ciProfileData* data, int flag __ store(flags, addr); } -void LIRGenerator::profile_null_free_array(LIRItem array, ciMethodData* md, ciProfileData* data) { +template void LIRGenerator::profile_null_free_array(LIRItem array, ciMethodData* md, ArrayData* data) { assert(compilation()->profile_array_accesses(), "array access profiling is disabled"); - LabelObj* L_end = new LabelObj(); LIR_Opr tmp = new_register(T_METADATA); __ check_null_free_array(array.result(), tmp); #ifdef RISCV @@ -3020,14 +3015,20 @@ void LIRGenerator::profile_null_free_array(LIRItem array, ciMethodData* md, ciPr // See LIR_Assembler::emit_opNullFreeArrayCheck __ cmp(lir_cond_equal, tmp, LIR_OprFact::metadataConst(nullptr)); #endif - profile_flags(md, data, ArrayStoreData::null_free_array_byte_constant(), lir_cond_equal); + profile_flags(md, data, ArrayData::null_free_array_byte_constant(), lir_cond_equal); } -template void LIRGenerator::profile_array_type(AccessIndexed* x, ciMethodData*& md, ArrayData*& load_store) { +void LIRGenerator::profile_array_type(AccessIndexed* x, ciMethodData*& md, ciArrayStoreData* store) { assert(compilation()->profile_array_accesses(), "array access profiling is disabled"); LIR_Opr mdp = LIR_OprFact::illegalOpr; - profile_type(md, md->byte_offset_of_slot(load_store, ArrayData::array_offset()), 0, - load_store->array()->type(), x->array(), mdp, true, nullptr, nullptr); + profile_type(md, md->byte_offset_of_slot(store, ArrayStoreData::array_offset()), 0, + store->array()->type(), x->array(), mdp, true, nullptr, nullptr); +} + +void LIRGenerator::profile_multiple_array_types(AccessIndexed* x, LIRItem array, ciMethodData* md, ciArrayLoadData* load) { + assert(compilation()->profile_array_accesses(), "array access profiling is disabled"); + + __ profile_multiple_array_types(md, load, array.result(), new_pointer_register(), new_pointer_register()); } void LIRGenerator::profile_element_type(Value element, ciMethodData* md, ciArrayLoadData* load_data) { diff --git a/src/hotspot/share/c1/c1_LIRGenerator.hpp b/src/hotspot/share/c1/c1_LIRGenerator.hpp index 65c64d2262c..322d220d7bb 100644 --- a/src/hotspot/share/c1/c1_LIRGenerator.hpp +++ b/src/hotspot/share/c1/c1_LIRGenerator.hpp @@ -493,8 +493,9 @@ class LIRGenerator: public InstructionVisitor, public BlockClosure { void profile_parameters(Base* x); void profile_parameters_at_call(ProfileCall* x); void profile_flags(ciMethodData* md, ciProfileData* load_store, int flag, LIR_Condition condition = lir_cond_always); - void profile_null_free_array(LIRItem array, ciMethodData* md, ciProfileData* load_store); - template void profile_array_type(AccessIndexed* x, ciMethodData*& md, ArrayData*& load_store); + template void profile_null_free_array(LIRItem array, ciMethodData* md, ArrayData* load_store); + void profile_array_type(AccessIndexed* x, ciMethodData*& md, ciArrayStoreData* store); + void profile_multiple_array_types(AccessIndexed* x, LIRItem array, ciMethodData* md, ciArrayLoadData* load); void profile_element_type(Value element, ciMethodData* md, ciArrayLoadData* load_store); bool profile_inline_klass(ciMethodData* md, ciProfileData* data, Value value, int flag); LIR_Opr mask_boolean(LIR_Opr array, LIR_Opr value, CodeEmitInfo*& null_check_info); diff --git a/src/hotspot/share/c1/c1_Runtime1.cpp b/src/hotspot/share/c1/c1_Runtime1.cpp index bcf55b454b6..97198c1cdc5 100644 --- a/src/hotspot/share/c1/c1_Runtime1.cpp +++ b/src/hotspot/share/c1/c1_Runtime1.cpp @@ -505,12 +505,6 @@ static void profile_flat_array(JavaThread* current, bool load, bool null_free) { ProfileData* data = md->bci_to_data(bci); assert(data != nullptr, "incorrect profiling entry"); if (data->is_ArrayLoadData()) { - assert(load, "should be an array load"); - ArrayLoadData* load_data = (ArrayLoadData*) data; - load_data->set_flat_array(); - if (null_free) { - load_data->set_null_free_array(); - } } else { assert(data->is_ArrayStoreData(), ""); assert(!load, "should be an array store"); diff --git a/src/hotspot/share/ci/ciCallProfile.hpp b/src/hotspot/share/ci/ciCallProfile.hpp index 76aca113169..d4cb0444826 100644 --- a/src/hotspot/share/ci/ciCallProfile.hpp +++ b/src/hotspot/share/ci/ciCallProfile.hpp @@ -26,6 +26,7 @@ #define SHARE_CI_CICALLPROFILE_HPP #include "ci/ciClassList.hpp" +#include "compiler/compiler_globals.hpp" #include "memory/allocation.hpp" // ciCallProfile @@ -61,18 +62,21 @@ class ciCallProfile : StackObj { int morphism() const { return _morphism; } int count() const { return _count; } - int receiver_count(int i) { + int receiver_count(int i) const { assert(i < _limit, "out of Call Profile MorphismLimit"); return _receiver_count[i]; } - float receiver_prob(int i) { + float receiver_prob(int i) const { assert(i < _limit, "out of Call Profile MorphismLimit"); return (float)_receiver_count[i]/(float)_count; } - ciKlass* receiver(int i) { + ciKlass* receiver(int i) const { assert(i < _limit, "out of Call Profile MorphismLimit"); return _receiver[i]; } + bool has_major_receiver() const { + return has_receiver(0) && (100.*receiver_prob(0) >= (float)TypeProfileMajorReceiverPercent); + } }; #endif // SHARE_CI_CICALLPROFILE_HPP diff --git a/src/hotspot/share/ci/ciClassList.hpp b/src/hotspot/share/ci/ciClassList.hpp index 04b13d5cae9..e5e964539c2 100644 --- a/src/hotspot/share/ci/ciClassList.hpp +++ b/src/hotspot/share/ci/ciClassList.hpp @@ -100,6 +100,7 @@ friend class ciExceptionHandlerStream; \ friend class ciObject; \ friend class ciNullObject; \ friend class ciInstance; \ +friend class ciMegamorphicTypeData; \ friend class ciMemberName; \ friend class ciMethod; \ friend class ciMethodData; \ diff --git a/src/hotspot/share/ci/ciMethod.cpp b/src/hotspot/share/ci/ciMethod.cpp index 7b4220ec636..ef2b256a144 100644 --- a/src/hotspot/share/ci/ciMethod.cpp +++ b/src/hotspot/share/ci/ciMethod.cpp @@ -452,13 +452,75 @@ int ciMethod::check_overflow(int c, Bytecodes::Code code) { return (c > 0 ? min_jint : c); // always non-positive } default: { - assert(Bytecodes::is_invoke(code), "%s", Bytecodes::name(code)); + assert(Bytecodes::is_invoke(code) || code == Bytecodes::_aaload, "%s", Bytecodes::name(code)); return (c < 0 ? max_jint : c); // always non-negative } } } +ciCallProfile ciMethod::megamorphic_profile_at_bci(int bci, int count, ciMegamorphicTypeData* megamorphic_type_data) { + ciCallProfile result; + + // In addition, virtual call sites have receiver type information + int receivers_count_total = 0; + int morphism = 0; + // Precompute morphism for the possible fixup + for (uint i = 0; i < megamorphic_type_data->row_limit(); i++) { + ciKlass* receiver = megamorphic_type_data->receiver(i); + if (receiver == nullptr) continue; + morphism++; + } + int epsilon = 0; + // For a call, it is assumed that either the type of the receiver(s) + // is recorded or an associated counter is incremented, but not both. With + // tiered compilation, however, both can happen due to the interpreter and + // C1 profiling invocations differently. Address that inconsistency here. + if (morphism == 1 && count > 0) { + epsilon = count; + count = 0; + } + for (uint i = 0; i < megamorphic_type_data->row_limit(); i++) { + ciKlass* receiver = megamorphic_type_data->receiver(i); + if (receiver == nullptr) continue; + int rcount = saturated_add(megamorphic_type_data->receiver_count(i), epsilon); + if (rcount == 0) rcount = 1; // Should be valid value + receivers_count_total = saturated_add(receivers_count_total, rcount); + // Add the receiver to result data. + result.add_receiver(receiver, rcount); + // If we extend profiling to record methods, + // we will set result._method also. + } + // Determine call site's morphism. + // The call site count is 0 with known morphism (only 1 or 2 receivers) + // or < 0 in the case of a type check failure for checkcast, aastore, instanceof. + // The call site count is > 0 in the case of a polymorphic virtual call. + if (morphism > 0 && morphism == result._limit) { + // The morphism <= MorphismLimit. + if ((morphism < ciCallProfile::MorphismLimit) || + (morphism == ciCallProfile::MorphismLimit && count == 0)) { +#ifdef ASSERT + if (count > 0) { + this->print_short_name(tty); + tty->print_cr(" @ bci:%d", bci); + this->print_codes(); + assert(false, "this call site should not be polymorphic"); + } +#endif + result._morphism = morphism; + } + } + // Make the count consistent if this is a call profile. If count is + // zero or less, presume that this is a typecheck profile and + // do nothing. Otherwise, increase count to be the sum of all + // receiver's counts. + if (count >= 0) { + count = saturated_add(count, receivers_count_total); + } + result._count = count; + return result; +} + // ------------------------------------------------------------------ // ciMethod::call_profile_at_bci // @@ -469,71 +531,25 @@ ciCallProfile ciMethod::call_profile_at_bci(int bci) { ciCallProfile result; if (method_data() != nullptr && method_data()->is_mature()) { ciProfileData* data = method_data()->bci_to_data(bci); - if (data != nullptr && data->is_CounterData()) { - // Every profiled call site has a counter. - int count = check_overflow(data->as_CounterData()->count(), java_code_at_bci(bci)); - - if (!data->is_ReceiverTypeData()) { - result._receiver_count[0] = 0; // that's a definite zero - } else { // ReceiverTypeData is a subclass of CounterData - ciReceiverTypeData* call = (ciReceiverTypeData*)data->as_ReceiverTypeData(); - // In addition, virtual call sites have receiver type information - int receivers_count_total = 0; - int morphism = 0; - // Precompute morphism for the possible fixup - for (uint i = 0; i < call->row_limit(); i++) { - ciKlass* receiver = call->receiver(i); - if (receiver == nullptr) continue; - morphism++; - } - int epsilon = 0; - // For a call, it is assumed that either the type of the receiver(s) - // is recorded or an associated counter is incremented, but not both. With - // tiered compilation, however, both can happen due to the interpreter and - // C1 profiling invocations differently. Address that inconsistency here. - if (morphism == 1 && count > 0) { - epsilon = count; - count = 0; - } - for (uint i = 0; i < call->row_limit(); i++) { - ciKlass* receiver = call->receiver(i); - if (receiver == nullptr) continue; - int rcount = saturated_add(call->receiver_count(i), epsilon); - if (rcount == 0) rcount = 1; // Should be valid value - receivers_count_total = saturated_add(receivers_count_total, rcount); - // Add the receiver to result data. - result.add_receiver(receiver, rcount); - // If we extend profiling to record methods, - // we will set result._method also. - } - // Determine call site's morphism. - // The call site count is 0 with known morphism (only 1 or 2 receivers) - // or < 0 in the case of a type check failure for checkcast, aastore, instanceof. - // The call site count is > 0 in the case of a polymorphic virtual call. - if (morphism > 0 && morphism == result._limit) { - // The morphism <= MorphismLimit. - if ((morphism < ciCallProfile::MorphismLimit) || - (morphism == ciCallProfile::MorphismLimit && count == 0)) { -#ifdef ASSERT - if (count > 0) { - this->print_short_name(tty); - tty->print_cr(" @ bci:%d", bci); - this->print_codes(); - assert(false, "this call site should not be polymorphic"); - } -#endif - result._morphism = morphism; - } - } - // Make the count consistent if this is a call profile. If count is - // zero or less, presume that this is a typecheck profile and - // do nothing. Otherwise, increase count to be the sum of all - // receiver's counts. - if (count >= 0) { - count = saturated_add(count, receivers_count_total); + if (data != nullptr) { + if (data->is_CounterData()) { + // Every profiled call site has a counter. + int count = check_overflow(data->as_CounterData()->count(), java_code_at_bci(bci)); + + if (!data->is_ReceiverTypeData()) { + result._receiver_count[0] = 0; // that's a definite zero + } else { // ReceiverTypeData is a subclass of CounterData + ciReceiverTypeData* call = (ciReceiverTypeData*) data->as_ReceiverTypeData(); + ciMegamorphicTypeData* megamorphic_type_data = call->megamorphic_type_data(); + return megamorphic_profile_at_bci(bci, count, megamorphic_type_data); } + result._count = count; + } else if (data->is_ArrayLoadData()) { + ciArrayLoadData* load = (ciArrayLoadData*) data->as_ArrayLoadData(); + int count = load->flat_count(); + ciMegamorphicTypeData* megamorphic_type_data = load->megamorphic_type_data(); + return megamorphic_profile_at_bci(bci, count, megamorphic_type_data); } - result._count = count; } } return result; @@ -678,12 +694,25 @@ bool ciMethod::array_access_profiled_type(int bci, ciKlass*& array_type, ciKlass ciProfileData* data = method_data()->bci_to_data(bci); if (data != nullptr) { if (data->is_ArrayLoadData()) { + // ciCallProfile profile = call_profile_at_bci(bci); ciArrayLoadData* array_access = (ciArrayLoadData*) data->as_ArrayLoadData(); - array_type = array_access->array()->valid_type(); + array_type = nullptr; //array_access->array()->valid_type(); element_type = array_access->element()->valid_type(); element_ptr = array_access->element()->ptr_kind(); - flat_array = array_access->flat_array(); - null_free_array = array_access->null_free_array(); + flat_array = false; + null_free_array = false; + array_type = nullptr; + // if (profile.morphism() == 1 && flat_array) { + // array_type = profile.receiver(0); + // } +#ifdef ASSERT + if (array_type != nullptr) { + bool flat = array_type->is_flat_array_klass(); + bool null_free = array_type->as_array_klass()->is_elem_null_free(); + assert(!flat || flat_array, "inconsistency"); + assert(!null_free || null_free_array, "inconsistency"); + } +#endif update_flags_from_type(array_type, flat_array, null_free_array); return true; } else if (data->is_ArrayStoreData()) { diff --git a/src/hotspot/share/ci/ciMethod.hpp b/src/hotspot/share/ci/ciMethod.hpp index 70f75c02c16..a91ec8e8017 100644 --- a/src/hotspot/share/ci/ciMethod.hpp +++ b/src/hotspot/share/ci/ciMethod.hpp @@ -37,6 +37,7 @@ #include "utilities/vmEnums.hpp" class ciMethodBlocks; +class ciMegamorphicTypeData; class MethodLiveness; class Arena; class BCEscapeAnalyzer; @@ -135,7 +136,7 @@ class ciMethod : public ciMetadata { // Check and update the profile counter in case of overflow static int check_overflow(int c, Bytecodes::Code code); - public: +public: void check_is_loaded() const { assert(is_loaded(), "not loaded"); } // Basic method information. @@ -266,6 +267,7 @@ class ciMethod : public ciMetadata { ciTypeFlow* get_flow_analysis(); ciTypeFlow* get_osr_flow_analysis(int osr_bci); // alternate entry point ciCallProfile call_profile_at_bci(int bci); + ciCallProfile megamorphic_profile_at_bci(int bci, int count, ciMegamorphicTypeData* megamorphic_type_data); // Does type profiling provide any useful information at this point? bool argument_profiled_type(int bci, int i, ciKlass*& type, ProfilePtrKind& ptr_kind); diff --git a/src/hotspot/share/ci/ciMethodData.cpp b/src/hotspot/share/ci/ciMethodData.cpp index fe1cce9dade..44f39d6dbb4 100644 --- a/src/hotspot/share/ci/ciMethodData.cpp +++ b/src/hotspot/share/ci/ciMethodData.cpp @@ -308,9 +308,10 @@ bool ciMethodData::load_data() { return true; } -void ciReceiverTypeData::translate_receiver_data_from(const ProfileData* data) { +bool ciMegamorphicTypeData::translate_type_data_from(const MegamorphicTypeData* data) { + bool cleared_row = false; for (uint row = 0; row < row_limit(); row++) { - Klass* k = data->as_ReceiverTypeData()->receiver(row); + Klass* k = data->receiver(row); if (k != nullptr && k->class_loader_data() != nullptr && is_klass_loaded(k)) { if (k->is_loader_alive()) { ciKlass* klass = CURRENT_ENV->get_klass(k); @@ -318,13 +319,28 @@ void ciReceiverTypeData::translate_receiver_data_from(const ProfileData* data) { } else { // With concurrent class unloading, the MDO could have stale metadata; override it clear_row(row); + cleared_row = true; } } else { set_receiver(row, nullptr); } } + return cleared_row; } +#ifndef PRODUCT +void ciMegamorphicTypeData::print_receiver_data_on(outputStream* st, int total) const { + total += count(); + for (uint row = 0; row < row_limit(); row++) { + if (receiver(row) != nullptr) { + _pd->tab(st); + receiver(row)->print_name_on(st); + st->print_cr("(%u %4.2f)", receiver_count(row), (float) receiver_count(row) / (float) total); + } + } +} +#endif + void ciTypeStackSlotEntries::translate_type_data_from(const TypeStackSlotEntries* entries) { for (int i = 0; i < number_of_entries(); i++) { intptr_t k = entries->type(i); @@ -834,8 +850,7 @@ void ciMethodData::dump_replay_data(outputStream* out) { dump_replay_data_receiver_type_helper(out, round, count, array_store_data); } else if (pdata->is_ArrayLoadData()) { ciArrayLoadData* array_load_data = (ciArrayLoadData*)pdata; - dump_replay_data_type_helper(out, round, count, array_load_data, ciArrayLoadData::array_offset(), - array_load_data->array()->valid_type()); + dump_replay_data_receiver_type_helper(out, round, count, array_load_data); dump_replay_data_type_helper(out, round, count, array_load_data, ciArrayLoadData::element_offset(), array_load_data->element()->valid_type()); } else if (pdata->is_ACmpData()) { @@ -951,19 +966,8 @@ void ciCallTypeData::print_data_on(outputStream* st, const char* extra) const { } void ciReceiverTypeData::print_receiver_data_on(outputStream* st) const { - uint row; - int entries = 0; - for (row = 0; row < row_limit(); row++) { - if (receiver(row) != nullptr) entries++; - } - st->print_cr("count(%u) entries(%u)", count(), entries); - for (row = 0; row < row_limit(); row++) { - if (receiver(row) != nullptr) { - tab(st); - receiver(row)->print_name_on(st); - st->print_cr("(%u)", receiver_count(row)); - } - } + st->print_cr("count(%u) entries(%u)", count(), _megamorphic_type_data.entries()); + megamorphic_type_data()->print_receiver_data_on(st, count()); } void ciReceiverTypeData::print_data_on(outputStream* st, const char* extra) const { @@ -1015,11 +1019,17 @@ void ciArrayStoreData::print_data_on(outputStream* st, const char* extra) const } void ciArrayLoadData::print_data_on(outputStream* st, const char* extra) const { - print_shared(st, "ciArrayLoadData", extra); + print_shared(st, "ArrayLoad", extra); + st->print("not flat %u (null free = %u, nullable = %u)", not_flat_count(), not_flat_null_free_count(), not_flat_nullable_count()); + st->cr(); + tab(st); + st->print("flat %u (null free atomic = %u, null free not atomic = %u, nullable = %u)", flat_count(), flat_nullfree_atomic_count(), flat_nullfree_not_atomic_count(), flat_nullable_count()); st->cr(); tab(st, true); st->print("array"); - array()->print_data_on(st); + tab(st); + megamorphic_type_data()->print_receiver_data_on(st, 0); + st->cr(); tab(st, true); st->print("element"); element()->print_data_on(st); diff --git a/src/hotspot/share/ci/ciMethodData.hpp b/src/hotspot/share/ci/ciMethodData.hpp index fca4f07099f..e88bcab0201 100644 --- a/src/hotspot/share/ci/ciMethodData.hpp +++ b/src/hotspot/share/ci/ciMethodData.hpp @@ -141,6 +141,28 @@ class ciSingleTypeEntry : public SingleTypeEntry, ciTypeEntries { #endif }; +class ciMegamorphicTypeData : public MegamorphicTypeData { +public: + bool translate_type_data_from(const MegamorphicTypeData* ret); + + void set_receiver(uint row, ciKlass* recv) { + assert((uint)row < row_limit(), "oob"); + set_intptr_at(receiver_cell_index(row), (intptr_t) recv); + } + + ciKlass* receiver(uint row) const { + assert((uint)row < row_limit(), "oob"); + ciKlass* recv = (ciKlass*)intptr_at(receiver_cell_index(row)); + assert(recv == nullptr || recv->is_klass(), "wrong type"); + return recv; + } + +#ifndef PRODUCT + void print_receiver_data_on(outputStream* st, int total) const; +#endif +}; + + class ciCallTypeData : public CallTypeData { public: ciCallTypeData(DataLayout* layout) : CallTypeData(layout) {} @@ -193,25 +215,26 @@ class ciCallTypeData : public CallTypeData { class ciReceiverTypeData : public ReceiverTypeData { public: ciReceiverTypeData(DataLayout* layout) : ReceiverTypeData(layout) {}; + ciMegamorphicTypeData* megamorphic_type_data() const { return (ciMegamorphicTypeData*)ReceiverTypeData::megamorphic_type_data(); } void set_receiver(uint row, ciKlass* recv) { assert((uint)row < row_limit(), "oob"); - set_intptr_at(receiver0_offset + row * receiver_type_row_cell_count, - (intptr_t) recv); + set_intptr_at(_megamorphic_type_data.receiver_cell_index(row), (intptr_t) recv); } ciKlass* receiver(uint row) const { - assert((uint)row < row_limit(), "oob"); - ciKlass* recv = (ciKlass*)intptr_at(receiver0_offset + row * receiver_type_row_cell_count); - assert(recv == nullptr || recv->is_klass(), "wrong type"); - return recv; + return megamorphic_type_data()->receiver(row); } // Copy & translate from oop based ReceiverTypeData virtual void translate_from(const ProfileData* data) { translate_receiver_data_from(data); } - void translate_receiver_data_from(const ProfileData* data); + void translate_receiver_data_from(const ProfileData* data) { + if (megamorphic_type_data()->translate_type_data_from(data->as_ReceiverTypeData()->megamorphic_type_data())) { + set_count(0); + } + } #ifndef PRODUCT void print_data_on(outputStream* st, const char* extra = nullptr) const; void print_receiver_data_on(outputStream* st) const; @@ -385,17 +408,27 @@ class ciArrayStoreData : public ArrayStoreData { }; class ciArrayLoadData : public ArrayLoadData { + public: ciArrayLoadData(DataLayout* layout) : ArrayLoadData(layout) {} - ciSingleTypeEntry* array() const { return (ciSingleTypeEntry*)ArrayLoadData::array(); } + // ciSingleTypeEntry* array() const { return (ciSingleTypeEntry*)ArrayLoadData::array(); } ciSingleTypeEntry* element() const { return (ciSingleTypeEntry*)ArrayLoadData::element(); } + ciMegamorphicTypeData* megamorphic_type_data() const { return (ciMegamorphicTypeData*)ArrayLoadData::megamorphic_type_data(); } virtual void translate_from(const ProfileData* data) { - array()->translate_type_data_from(data->as_ArrayLoadData()->array()); + if (megamorphic_type_data()->translate_type_data_from(data->as_ArrayLoadData()->megamorphic_type_data())) { + set_flat_nullable_count(0); + set_flat_nullfree_atomic_count(0); + set_flat_nullfree_not_atomic_count(0); + } element()->translate_type_data_from(data->as_ArrayLoadData()->element()); } + ciKlass* receiver(uint row) { + return megamorphic_type_data()->receiver(row); + } + #ifndef PRODUCT void print_data_on(outputStream* st, const char* extra = nullptr) const; #endif diff --git a/src/hotspot/share/compiler/compiler_globals.hpp b/src/hotspot/share/compiler/compiler_globals.hpp index e1f8d9f8922..7062f40992c 100644 --- a/src/hotspot/share/compiler/compiler_globals.hpp +++ b/src/hotspot/share/compiler/compiler_globals.hpp @@ -383,6 +383,10 @@ "If compilation is stopped with an error, capture diagnostic " \ "information at the bailout point") \ \ + product(intx, TypeProfileMajorReceiverPercent, 90, \ + "% of major receiver type to all profiled receivers") \ + range(0, 100) \ + \ // end of COMPILER_FLAGS DECLARE_FLAGS(COMPILER_FLAGS) diff --git a/src/hotspot/share/oops/flatArrayKlass.cpp b/src/hotspot/share/oops/flatArrayKlass.cpp index 815ba104bc2..e562a961470 100644 --- a/src/hotspot/share/oops/flatArrayKlass.cpp +++ b/src/hotspot/share/oops/flatArrayKlass.cpp @@ -392,7 +392,8 @@ void FlatArrayKlass::print_value_on(outputStream* st) const { assert(is_klass(), "must be klass"); element_klass()->print_value_on(st); - st->print("[]"); + st->print("[] "); + LayoutKindHelper::print_on(layout_kind(), st); } #ifndef PRODUCT diff --git a/src/hotspot/share/oops/methodData.cpp b/src/hotspot/share/oops/methodData.cpp index f3ff17cd9ca..c7ce56403d8 100644 --- a/src/hotspot/share/oops/methodData.cpp +++ b/src/hotspot/share/oops/methodData.cpp @@ -447,8 +447,8 @@ void VirtualCallTypeData::print_data_on(outputStream* st, const char* extra) con // that the check is reached, and a series of (Klass*, count) pairs // which are used to store a type profile for the receiver of the check. -void ReceiverTypeData::clean_weak_klass_links(bool always_clean) { - for (uint row = 0; row < row_limit(); row++) { +void MegamorphicTypeData::clean_weak_klass_links(bool always_clean) { + for (uint row = 0; row < row_limit(); row++) { Klass* p = receiver(row); if (p != nullptr) { if (!always_clean && p->is_instance_klass() && InstanceKlass::cast(p)->is_not_initialized()) { @@ -461,34 +461,53 @@ void ReceiverTypeData::clean_weak_klass_links(bool always_clean) { } } -void ReceiverTypeData::metaspace_pointers_do(MetaspaceClosure *it) { +void ReceiverTypeData::clean_weak_klass_links(bool always_clean) { + _megamorphic_type_data.clean_weak_klass_links(always_clean); +} + +void MegamorphicTypeData::metaspace_pointers_do(MetaspaceClosure *it) { for (uint row = 0; row < row_limit(); row++) { - Klass** recv = (Klass**)intptr_at_adr(receiver_cell_index(row)); + Klass** recv = (Klass**)_pd->intptr_at_adr(receiver_cell_index(row)); it->push(recv); } } -void ReceiverTypeData::print_receiver_data_on(outputStream* st) const { - uint row; +void ReceiverTypeData::metaspace_pointers_do(MetaspaceClosure *it) { + _megamorphic_type_data.metaspace_pointers_do(it); +} + +int MegamorphicTypeData::entries() const { int entries = 0; - for (row = 0; row < row_limit(); row++) { + for (uint row = 0; row < row_limit(); row++) { if (receiver(row) != nullptr) entries++; } - st->print_cr("count(%u) entries(%u)", count(), entries); - int total = count(); - for (row = 0; row < row_limit(); row++) { + return entries; +} + +int MegamorphicTypeData::count() const { + int total = 0; + for (uint row = 0; row < row_limit(); row++) { if (receiver(row) != nullptr) { total += receiver_count(row); } } - for (row = 0; row < row_limit(); row++) { + return total; +} + +void MegamorphicTypeData::print_receiver_data_on(outputStream* st, int total) const { + total += count(); + for (uint row = 0; row < row_limit(); row++) { if (receiver(row) != nullptr) { - tab(st); + _pd->tab(st); receiver(row)->print_value_on(st); st->print_cr("(%u %4.2f)", receiver_count(row), (float) receiver_count(row) / (float) total); } } } +void ReceiverTypeData::print_receiver_data_on(outputStream* st) const { + st->print_cr("count(%u) entries(%u)", count(), _megamorphic_type_data.entries()); + _megamorphic_type_data.print_receiver_data_on(st, count()); +} void ReceiverTypeData::print_data_on(outputStream* st, const char* extra) const { print_shared(st, "ReceiverTypeData", extra); print_receiver_data_on(st); @@ -729,10 +748,16 @@ void ArrayStoreData::print_data_on(outputStream* st, const char* extra) const { void ArrayLoadData::print_data_on(outputStream* st, const char* extra) const { print_shared(st, "ArrayLoad", extra); + st->print("not flat %u (null free = %u, nullable = %u)", not_flat_count(), not_flat_null_free_count(), not_flat_nullable_count()); + st->cr(); + tab(st); + st->print("flat %u (null free atomic = %u, null free not atomic = %u, nullable = %u)", flat_count(), flat_nullfree_atomic_count(), flat_nullfree_not_atomic_count(), flat_nullable_count()); st->cr(); tab(st, true); st->print("array"); - _array.print_data_on(st); + tab(st); + _megamorphic_type_data.print_receiver_data_on(st, 0); + st->cr(); tab(st, true); st->print("element"); _element.print_data_on(st); diff --git a/src/hotspot/share/oops/methodData.hpp b/src/hotspot/share/oops/methodData.hpp index 19ea9ac3ea1..52143fadad3 100644 --- a/src/hotspot/share/oops/methodData.hpp +++ b/src/hotspot/share/oops/methodData.hpp @@ -311,6 +311,7 @@ class ProfileData : public ResourceObj { friend class TypeEntries; friend class SingleTypeEntry; friend class TypeStackSlotEntries; + friend class MegamorphicTypeData; private: enum { tab_width_one = 16, @@ -1160,42 +1161,62 @@ class CallTypeData : public CounterData { // // Updated by platform-specific code, for example MacroAssembler::profile_receiver_type. // -class ReceiverTypeData : public CounterData { + +class MegamorphicTypeData { friend class VMStructs; protected: enum { - receiver0_offset = counter_cell_count, + receiver0_offset, count0_offset, receiver_type_row_cell_count = (count0_offset + 1) - receiver0_offset }; -public: - ReceiverTypeData(DataLayout* layout) : CounterData(layout) { - assert(layout->tag() == DataLayout::receiver_type_data_tag || - layout->tag() == DataLayout::virtual_call_data_tag || - layout->tag() == DataLayout::virtual_call_type_data_tag || - layout->tag() == DataLayout::array_store_data_tag, "wrong type"); + ProfileData* _pd; + const int _base_off; + const int _type_width; + + void set_uint_at(int index, uint value) { + _pd->set_uint_at(index, value); + } + uint uint_at(int index) const { + return _pd->uint_at(index); } - virtual bool is_ReceiverTypeData() const { return true; } +protected: - static int static_cell_count() { - return counter_cell_count + (uint) TypeProfileWidth * receiver_type_row_cell_count; + void set_intptr_at(int index, intptr_t value) { + _pd->set_intptr_at(index, value); } - virtual int cell_count() const { - return static_cell_count(); + intptr_t intptr_at(int index) const { + return _pd->intptr_at(index); } - // Direct accessors - static uint row_limit() { - return (uint) TypeProfileWidth; +public: + + MegamorphicTypeData(ProfileData* pd, int base_off, int type_width) + : _pd(pd), _base_off(base_off), _type_width(type_width) {} + + static int static_cell_count(int type_width) { + return type_width * receiver_type_row_cell_count; + } + int cell_count() const { + return _type_width * receiver_type_row_cell_count; + } + uint row_limit() const { + return (uint)_type_width; + } + static int static_receiver_cell_index(int base, uint row) { + return base + receiver0_offset + row * receiver_type_row_cell_count; + } + static int static_receiver_count_cell_index(int base, uint row) { + return base + count0_offset + row * receiver_type_row_cell_count; } - static int receiver_cell_index(uint row) { - return receiver0_offset + row * receiver_type_row_cell_count; + int receiver_cell_index(uint row) const { + return static_receiver_cell_index(_base_off, row); } - static int receiver_count_cell_index(uint row) { - return count0_offset + row * receiver_type_row_cell_count; + int receiver_count_cell_index(uint row) const { + return static_receiver_count_cell_index(_base_off, row); } Klass* receiver(uint row) const { @@ -1223,6 +1244,94 @@ class ReceiverTypeData : public CounterData { void clear_row(uint row) { assert(row < row_limit(), "oob"); + set_receiver(row, nullptr); + set_receiver_count(row, 0); + } + + // Code generation support + static ByteSize receiver_offset(int base, uint row) { + return ProfileData::cell_offset(static_receiver_cell_index(base, row)); + } + static ByteSize receiver_count_offset(int base, uint row) { + return ProfileData::cell_offset(static_receiver_count_cell_index(base, row)); + } + // static ByteSize receiver_type_data_size() { + // return cell_offset(static_cell_count()); + // } + // + // GC support + void clean_weak_klass_links(bool always_clean); + + // CDS support + void metaspace_pointers_do(MetaspaceClosure* it); + + int entries() const; + int count() const; + void print_receiver_data_on(outputStream* st, int total) const; +}; + +class ReceiverTypeData : public CounterData { + friend class VMStructs; + friend class JVMCIVMStructs; +protected: + + MegamorphicTypeData _megamorphic_type_data; + +public: + ReceiverTypeData(DataLayout* layout) : CounterData(layout), + _megamorphic_type_data(this, base_of_megamorphic_type_data(), TypeProfileWidth) { + assert(layout->tag() == DataLayout::receiver_type_data_tag || + layout->tag() == DataLayout::virtual_call_data_tag || + layout->tag() == DataLayout::virtual_call_type_data_tag || + layout->tag() == DataLayout::array_store_data_tag || + layout->tag() == DataLayout::array_load_data_tag, "wrong type"); + } + + static int base_of_megamorphic_type_data() { + return counter_cell_count; + } + + const MegamorphicTypeData* megamorphic_type_data() const { + return &_megamorphic_type_data; + } + virtual bool is_ReceiverTypeData() const { return true; } + + static int static_cell_count() { + return counter_cell_count + MegamorphicTypeData::static_cell_count(TypeProfileWidth); + } + + virtual int cell_count() const { + return static_cell_count(); + } + + // Direct accessors + static uint row_limit() { + return (uint) TypeProfileWidth; + } + int receiver_cell_index(uint row) const { + return _megamorphic_type_data.receiver_cell_index(row); + } + int receiver_count_cell_index(uint row) const { + return _megamorphic_type_data.receiver_count_cell_index(row); + } + + Klass* receiver(uint row) const { + return _megamorphic_type_data.receiver(row); + } + + void set_receiver(uint row, Klass* k) { + _megamorphic_type_data.set_receiver(row, k); + } + + uint receiver_count(uint row) const { + return _megamorphic_type_data.receiver_count(row); + } + + void set_receiver_count(uint row, uint count) { + _megamorphic_type_data.set_receiver_count(row, count); + } + + void clear_row(uint row) { // Clear total count - indicator of polymorphic call site. // The site may look like as monomorphic after that but // it allow to have more accurate profiling information because @@ -1240,16 +1349,15 @@ class ReceiverTypeData : public CounterData { // We do sorting a profiling info (ciCallProfile) for compilation. // set_count(0); - set_receiver(row, nullptr); - set_receiver_count(row, 0); + _megamorphic_type_data.clear_row(row); } // Code generation support static ByteSize receiver_offset(uint row) { - return cell_offset(receiver_cell_index(row)); + return cell_offset(MegamorphicTypeData::static_receiver_cell_index(base_of_megamorphic_type_data(), row)); } static ByteSize receiver_count_offset(uint row) { - return cell_offset(receiver_count_cell_index(row)); + return cell_offset(MegamorphicTypeData::static_receiver_count_cell_index(base_of_megamorphic_type_data(), row)); } static ByteSize receiver_type_data_size() { return cell_offset(static_cell_count()); @@ -1976,74 +2084,169 @@ class ArrayStoreData : public ReceiverTypeData { virtual void print_data_on(outputStream* st, const char* extra = nullptr) const; }; -class ArrayLoadData : public BitData { +class ArrayLoadData : public ProfileData { private: enum { - flat_array_flag = BitData::last_bit_data_flag, - null_free_array_flag = flat_array_flag + 1, + not_flat_null_free_count_off_in_extra_cells, + not_flat_nullable_count_off_in_extra_cells, + flat_nullable_count_off_in_extra_cells, + flat_nullfree_atomic_count_off_in_extra_cells, + flat_nullfree_not_atomic_count_off_in_extra_cells, + extra_cells_count }; - SingleTypeEntry _array; SingleTypeEntry _element; + MegamorphicTypeData _megamorphic_type_data; + + static int extra_cells_off() { + return SingleTypeEntry::static_cell_count() + MegamorphicTypeData::static_cell_count(TypeProfileWidth); + } + + static int not_flat_null_free_count_off() { + return extra_cells_off() + not_flat_null_free_count_off_in_extra_cells; + } + + static int not_flat_nullable_count_off() { + return extra_cells_off() + not_flat_nullable_count_off_in_extra_cells; + } + + static int flat_nullable_count_off() { + return extra_cells_off() + flat_nullable_count_off_in_extra_cells; + } + + static int flat_nullfree_atomic_count_off() { + return extra_cells_off() + flat_nullfree_atomic_count_off_in_extra_cells; + } + + static int flat_nullfree_not_atomic_count_off() { + return extra_cells_off() + flat_nullfree_not_atomic_count_off_in_extra_cells; + } public: ArrayLoadData(DataLayout* layout) : - BitData(layout), - _array(0), - _element(SingleTypeEntry::static_cell_count()) { + ProfileData(layout), + _element(0), + _megamorphic_type_data(this, base_of_megamorphic_type_data(), TypeProfileWidth) { assert(layout->tag() == DataLayout::array_load_data_tag, "wrong type"); - _array.set_profile_data(this); _element.set_profile_data(this); } - const SingleTypeEntry* array() const { - return &_array; + static int base_of_megamorphic_type_data() { + return SingleTypeEntry::static_cell_count(); } const SingleTypeEntry* element() const { return &_element; } + const MegamorphicTypeData* megamorphic_type_data() const { + return &_megamorphic_type_data; + } + virtual bool is_ArrayLoadData() const { return true; } static int static_cell_count() { - return SingleTypeEntry::static_cell_count() * 2; + return extra_cells_off() + extra_cells_count; } virtual int cell_count() const { return static_cell_count(); } - void set_flat_array() { set_flag_at(flat_array_flag); } - bool flat_array() const { return flag_at(flat_array_flag); } + int not_flat_count() const { + return saturated_add(not_flat_null_free_count(), not_flat_nullable_count()); + } - void set_null_free_array() { set_flag_at(null_free_array_flag); } - bool null_free_array() const { return flag_at(null_free_array_flag); } + int flat_count() const { + return saturated_add(saturated_add(flat_nullfree_atomic_count(), flat_nullfree_not_atomic_count()), flat_nullable_count()); + } - // Code generation support - static int flat_array_byte_constant() { - return flag_number_to_constant(flat_array_flag); + int nullable_count() const { + return saturated_add(flat_nullable_count(), not_flat_nullable_count()); } - static int null_free_array_byte_constant() { - return flag_number_to_constant(null_free_array_flag); + int null_free_count() const { + return saturated_add(saturated_add(flat_nullfree_atomic_count(), flat_nullfree_not_atomic_count()), not_flat_null_free_count()); } - static ByteSize array_offset() { - return cell_offset(0); + int not_flat_null_free_count() const { + return uint_at(not_flat_null_free_count_off()); + } + + int not_flat_nullable_count() const { + return uint_at(not_flat_nullable_count_off()); + } + + int flat_nullable_count() const { + return uint_at(flat_nullable_count_off()); + } + + int flat_nullfree_atomic_count() const { + return uint_at(flat_nullfree_atomic_count_off()); + } + + int flat_nullfree_not_atomic_count() const { + return uint_at(flat_nullfree_not_atomic_count_off()); + } + + void set_flat_nullable_count(uint count) { + set_uint_at(flat_nullable_count_off(), count); + } + + void set_flat_nullfree_atomic_count(uint count) { + set_uint_at(flat_nullfree_atomic_count_off(), count); + } + + void set_flat_nullfree_not_atomic_count(uint count) { + set_uint_at(flat_nullfree_not_atomic_count_off(), count); + } + + static uint row_limit() { + return (uint) TypeProfileWidth; } + // Code generation support static ByteSize element_offset() { - return cell_offset(SingleTypeEntry::static_cell_count()); + return cell_offset(0); + } + + static ByteSize not_flat_null_free_count_offset() { + return cell_offset(not_flat_null_free_count_off()); + } + + static ByteSize not_flat_nullable_count_offset() { + return cell_offset(not_flat_nullable_count_off()); + } + + static ByteSize flat_nullable_count_offset() { + return cell_offset(flat_nullable_count_off()); + } + + static ByteSize flat_nullfree_atomic_count_offset() { + return cell_offset(flat_nullfree_atomic_count_off()); + } + + static ByteSize flat_nullfree_not_atomic_count_offset() { + return cell_offset(flat_nullfree_not_atomic_count_off()); + } + + static ByteSize receiver_offset(uint row) { + return cell_offset(MegamorphicTypeData::static_receiver_cell_index(base_of_megamorphic_type_data(), row)); + } + static ByteSize receiver_count_offset(uint row) { + return cell_offset(MegamorphicTypeData::static_receiver_count_cell_index(base_of_megamorphic_type_data(), row)); } + // static ByteSize receiver_type_data_size() { + // return cell_offset(static_cell_count()); + // } virtual void clean_weak_klass_links(bool always_clean) { - _array.clean_weak_klass_links(always_clean); + _megamorphic_type_data.clean_weak_klass_links(always_clean); _element.clean_weak_klass_links(always_clean); } virtual void metaspace_pointers_do(MetaspaceClosure* it) { - _array.metaspace_pointers_do(it); + _megamorphic_type_data.metaspace_pointers_do(it); _element.metaspace_pointers_do(it); } diff --git a/src/hotspot/share/opto/c2_globals.hpp b/src/hotspot/share/opto/c2_globals.hpp index 9ff9673d52f..b37b532c340 100644 --- a/src/hotspot/share/opto/c2_globals.hpp +++ b/src/hotspot/share/opto/c2_globals.hpp @@ -695,10 +695,6 @@ develop(bool, InlineAccessors, true, \ "inline accessor methods (get/set)") \ \ - product(intx, TypeProfileMajorReceiverPercent, 90, \ - "% of major receiver type to all profiled receivers") \ - range(0, 100) \ - \ product(bool, PrintIntrinsics, false, DIAGNOSTIC, \ "prints attempted and successful inlining of intrinsics") \ \ diff --git a/src/hotspot/share/opto/compile.cpp b/src/hotspot/share/opto/compile.cpp index 7c1fb53a55b..e556f660cbe 100644 --- a/src/hotspot/share/opto/compile.cpp +++ b/src/hotspot/share/opto/compile.cpp @@ -3195,6 +3195,7 @@ void Compile::Optimize() { // Loop transforms on the ideal graph. Range Check Elimination, // peeling, unrolling, etc. + bool split_if_progress = false;; // Set loop opts counter if((_loop_opts_cnt > 0) && (has_loops() || has_split_ifs())) { { @@ -3204,6 +3205,7 @@ void Compile::Optimize() { if (major_progress()) print_method(PHASE_PHASEIDEALLOOP1, 2); if (failing()) return; } + split_if_progress = major_progress() && !has_loops(); // Loop opts pass if partial peeling occurred in previous pass if(PartialPeelLoop && major_progress() && (_loop_opts_cnt > 0)) { TracePhase tp(_t_idealLoop); @@ -3248,6 +3250,10 @@ void Compile::Optimize() { if (failing()) return; + if (split_if_progress && !major_progress()) { + set_major_progress(); + } + // Loop transforms on the ideal graph. Range Check Elimination, // peeling, unrolling, etc. if (!optimize_loops(igvn, LoopOptsDefault)) { diff --git a/src/hotspot/share/opto/doCall.cpp b/src/hotspot/share/opto/doCall.cpp index 4dc5404e598..fefff0f3354 100644 --- a/src/hotspot/share/opto/doCall.cpp +++ b/src/hotspot/share/opto/doCall.cpp @@ -234,7 +234,7 @@ CallGenerator* Compile::call_generator(ciMethod* callee, int vtable_index, bool // Try using the type profile. if (call_does_dispatch && site_count > 0 && UseTypeProfile) { // The major receiver's count >= TypeProfileMajorReceiverPercent of site_count. - bool have_major_receiver = profile.has_receiver(0) && (100.*profile.receiver_prob(0) >= (float)TypeProfileMajorReceiverPercent); + bool have_major_receiver = profile.has_major_receiver(); ciMethod* receiver_method = nullptr; int morphism = profile.morphism(); diff --git a/src/hotspot/share/opto/graphKit.cpp b/src/hotspot/share/opto/graphKit.cpp index 64ec4468d34..7d86d1e309f 100644 --- a/src/hotspot/share/opto/graphKit.cpp +++ b/src/hotspot/share/opto/graphKit.cpp @@ -3864,8 +3864,8 @@ Node* GraphKit::gen_checkcast(Node* obj, Node* superklass, Node** failure_contro } else if (array_obj->is_Phi()) { Node* region = array_obj->in(0); // TODO make this more robust (see JDK-8231346) - if (region->req() == 3 && region->in(2) != nullptr && region->in(2)->in(0) != nullptr) { - IfNode* iff = region->in(2)->in(0)->isa_If(); + if (region->req() == 3 && region->in(1) != nullptr && region->in(1)->in(0) != nullptr) { + IfNode* iff = region->in(1)->in(0)->isa_If(); if (iff != nullptr) { iff->is_flat_array_check(&_gvn, &array); } diff --git a/src/hotspot/share/opto/inlinetypenode.cpp b/src/hotspot/share/opto/inlinetypenode.cpp index 0a5939ba7bf..94c8f96455b 100644 --- a/src/hotspot/share/opto/inlinetypenode.cpp +++ b/src/hotspot/share/opto/inlinetypenode.cpp @@ -1397,7 +1397,7 @@ InlineTypeNode* InlineTypeNode::make_from_flat_impl(GraphKit* kit, ciInlineKlass return LoadFlatNode::load(kit, vk, base, ptr, null_free, trust_null_free_oop, decorators); } -InlineTypeNode* InlineTypeNode::make_from_flat_array(GraphKit* kit, ciInlineKlass* vk, Node* base, Node* idx) { +InlineTypeNode* InlineTypeNode::make_from_flat_array(GraphKit* kit, ciInlineKlass* vk, Node* base, Node* idx, float null_free_prob,float null_free_atomic_prob) { assert(vk->maybe_flat_in_array(), "element type %s cannot be flat in array", vk->name()->as_utf8()); PhaseGVN& gvn = kit->gvn(); // The flat field loads are dependent on both the array layout checks as well as the range check. @@ -1406,6 +1406,7 @@ InlineTypeNode* InlineTypeNode::make_from_flat_array(GraphKit* kit, ciInlineKlas InlineTypeNode* vt_nullable = nullptr; InlineTypeNode* vt_null_free = nullptr; InlineTypeNode* vt_non_atomic = nullptr; + Node *top = kit->C->top(); RegionNode* region = new RegionNode(4); gvn.set_type(region, Type::CONTROL); @@ -1423,20 +1424,25 @@ InlineTypeNode* InlineTypeNode::make_from_flat_array(GraphKit* kit, ciInlineKlas kit->record_for_igvn(io); Node* bol_null_free = kit->null_free_array_test(base); // Argument evaluation order is undefined in C++ and since this sets control, it needs to come first - IfNode* iff_null_free = kit->create_and_map_if(kit->control(), bol_null_free, PROB_FAIR, COUNT_UNKNOWN); + IfNode* iff_null_free = kit->create_and_map_if(kit->control(), bol_null_free, clamp(null_free_prob, PROB_MIN, PROB_MAX), COUNT_UNKNOWN); // Nullable kit->set_control(kit->IfFalse(iff_null_free)); if (!kit->stopped()) { assert(vk->has_nullable_atomic_layout(), "element type %s does not have a nullable flat layout", vk->name()->as_utf8()); kit->set_all_memory(input_memory_state); - Node* cast = kit->cast_to_flat_array_exact(base, vk, false, true); - Node* ptr = kit->array_element_address(cast, idx, T_FLAT_ELEMENT); - vt_nullable = InlineTypeNode::make_from_flat(kit, vk, cast, ptr, true, false, false, decorators); + if (null_free_prob == 1 && !kit->too_many_traps_or_recompiles(Deoptimization::Reason_class_check)) { + PreserveJVMState pjvms(kit); + kit->uncommon_trap_exact(Deoptimization::Reason_class_check, Deoptimization::Action_maybe_recompile); + } else { + Node* cast = kit->cast_to_flat_array_exact(base, vk, false, true); + Node* ptr = kit->array_element_address(cast, idx, T_FLAT_ELEMENT); + vt_nullable = InlineTypeNode::make_from_flat(kit, vk, cast, ptr, true, false, false, decorators); - region->init_req(1, kit->control()); - mem->set_req(1, kit->reset_memory()); - io->set_req(1, kit->i_o()); + region->init_req(1, kit->control()); + mem->set_req(1, kit->reset_memory()); + io->set_req(1, kit->i_o()); + } } // Null-free @@ -1444,35 +1450,52 @@ InlineTypeNode* InlineTypeNode::make_from_flat_array(GraphKit* kit, ciInlineKlas if (!kit->stopped()) { kit->set_all_memory(input_memory_state); - Node* bol_atomic = kit->null_free_atomic_array_test(base, vk); - IfNode* iff_atomic = kit->create_and_map_if(kit->control(), bol_atomic, PROB_FAIR, COUNT_UNKNOWN); - - // Atomic - kit->set_control(kit->IfTrue(iff_atomic)); - if (!kit->stopped()) { - assert(vk->has_null_free_atomic_layout(), "element type %s does not have a null-free atomic flat layout", vk->name()->as_utf8()); - kit->set_all_memory(input_memory_state); - Node* cast = kit->cast_to_flat_array_exact(base, vk, true, true); - Node* ptr = kit->array_element_address(cast, idx, T_FLAT_ELEMENT); - vt_null_free = InlineTypeNode::make_from_flat(kit, vk, cast, ptr, true, false, true, decorators); + if (null_free_prob == 0 && !kit->too_many_traps_or_recompiles(Deoptimization::Reason_class_check)) { + PreserveJVMState pjvms(kit); + kit->uncommon_trap_exact(Deoptimization::Reason_class_check, Deoptimization::Action_maybe_recompile); + } else { + Node *bol_atomic = kit->null_free_atomic_array_test(base, vk); + IfNode *iff_atomic = kit->create_and_map_if(kit->control(), bol_atomic, + clamp(null_free_atomic_prob, PROB_MIN, PROB_MAX), COUNT_UNKNOWN); - region->init_req(2, kit->control()); - mem->set_req(2, kit->reset_memory()); - io->set_req(2, kit->i_o()); - } + // Atomic + kit->set_control(kit->IfTrue(iff_atomic)); + if (!kit->stopped()) { + assert(vk->has_null_free_atomic_layout(), "element type %s does not have a null-free atomic flat layout", + vk->name()->as_utf8()); + kit->set_all_memory(input_memory_state); + if (null_free_atomic_prob == 0 && !kit->too_many_traps_or_recompiles(Deoptimization::Reason_class_check)) { + PreserveJVMState pjvms(kit); + kit->uncommon_trap_exact(Deoptimization::Reason_class_check, Deoptimization::Action_maybe_recompile); + } else { + Node *cast = kit->cast_to_flat_array_exact(base, vk, true, true); + Node *ptr = kit->array_element_address(cast, idx, T_FLAT_ELEMENT); + vt_null_free = InlineTypeNode::make_from_flat(kit, vk, cast, ptr, true, false, true, decorators); + region->init_req(2, kit->control()); + mem->set_req(2, kit->reset_memory()); + io->set_req(2, kit->i_o()); + } + } - // Non-Atomic - kit->set_control(kit->IfFalse(iff_atomic)); - if (!kit->stopped()) { - assert(vk->has_null_free_non_atomic_layout(), "element type %s does not have a null-free non-atomic flat layout", vk->name()->as_utf8()); - kit->set_all_memory(input_memory_state); - Node* cast = kit->cast_to_flat_array_exact(base, vk, true, false); - Node* ptr = kit->array_element_address(cast, idx, T_FLAT_ELEMENT); - vt_non_atomic = InlineTypeNode::make_from_flat(kit, vk, cast, ptr, false, false, true, decorators); + // Non-Atomic + kit->set_control(kit->IfFalse(iff_atomic)); + if (!kit->stopped()) { + assert(vk->has_null_free_non_atomic_layout(), + "element type %s does not have a null-free non-atomic flat layout", vk->name()->as_utf8()); + kit->set_all_memory(input_memory_state); + if (null_free_atomic_prob == 1 && !kit->too_many_traps_or_recompiles(Deoptimization::Reason_class_check)) { + PreserveJVMState pjvms(kit); + kit->uncommon_trap_exact(Deoptimization::Reason_class_check, Deoptimization::Action_maybe_recompile); + } else { + Node *cast = kit->cast_to_flat_array_exact(base, vk, true, false); + Node *ptr = kit->array_element_address(cast, idx, T_FLAT_ELEMENT); + vt_non_atomic = InlineTypeNode::make_from_flat(kit, vk, cast, ptr, false, false, true, decorators); - region->init_req(3, kit->control()); - mem->set_req(3, kit->reset_memory()); - io->set_req(3, kit->i_o()); + region->init_req(3, kit->control()); + mem->set_req(3, kit->reset_memory()); + io->set_req(3, kit->i_o()); + } + } } } diff --git a/src/hotspot/share/opto/inlinetypenode.hpp b/src/hotspot/share/opto/inlinetypenode.hpp index 44eba6fbc72..4d48d7f08e6 100644 --- a/src/hotspot/share/opto/inlinetypenode.hpp +++ b/src/hotspot/share/opto/inlinetypenode.hpp @@ -88,7 +88,7 @@ class InlineTypeNode : public TypeNode { // Create and initialize by loading the field values from a flat field or array static InlineTypeNode* make_from_flat(GraphKit* kit, ciInlineKlass* vk, Node* base, Node* ptr, bool atomic, bool immutable_memory, bool null_free, DecoratorSet decorators); - static InlineTypeNode* make_from_flat_array(GraphKit* kit, ciInlineKlass* vk, Node* base, Node* idx); + static InlineTypeNode* make_from_flat_array(GraphKit* kit, ciInlineKlass* vk, Node* base, Node* idx, float null_free_prob = PROB_FAIR, float null_free_atomic_prob = PROB_FAIR); // Create and initialize with the inputs or outputs of a MultiNode (method entry or call) static InlineTypeNode* make_from_multi(GraphKit* kit, MultiNode* multi, ciInlineKlass* vk, uint& base_input, bool in, bool null_free = true); // Create with null field values diff --git a/src/hotspot/share/opto/parse.hpp b/src/hotspot/share/opto/parse.hpp index a1411ac4770..f2607c14c6f 100644 --- a/src/hotspot/share/opto/parse.hpp +++ b/src/hotspot/share/opto/parse.hpp @@ -146,6 +146,7 @@ class InlineTree : public AnyObj { //------------------------------Parse------------------------------------------ // Parse bytecodes, build a Graph class Parse : public GraphKit { + friend class ArrayLoad; public: // Per-block information needed by the parser: class Block { diff --git a/src/hotspot/share/opto/parse2.cpp b/src/hotspot/share/opto/parse2.cpp index 91845f605d4..ac55bace002 100644 --- a/src/hotspot/share/opto/parse2.cpp +++ b/src/hotspot/share/opto/parse2.cpp @@ -22,6 +22,7 @@ * */ +#include "ci/ciFlatArrayKlass.hpp" #include "ci/ciInlineKlass.hpp" #include "ci/ciMethodData.hpp" #include "ci/ciSymbols.hpp" @@ -72,9 +73,629 @@ Node* Parse::record_profile_for_speculation_at_array_load(Node* ld) { return ld; } +class ArrayLoad { +private: + BasicType _bt; + Parse& _parse; + PhaseGVN& _gvn; + const Type* _elemtype; + Node* _array_index; + Node* _array; + + Node* _region; + Node* _res_phi; + Node* _io_phi; + Node* _mem_phi; + Node* _mem; + Node* _io; + + bool emit_null_and_range_checks() { + _array = _parse.null_check(_array, T_ARRAY); + // Compile-time detect of null-exception? + if (_parse.stopped()) return true; // _kit.top(); + + const TypeAryPtr* arytype = _gvn.type(_array)->is_aryptr(); + const TypeInt* sizetype = arytype->size(); + _elemtype = arytype->elem(); + + if (UseUniqueSubclasses) { + const Type* el = _elemtype->make_ptr(); + if (el && el->isa_instptr()) { + const TypeInstPtr* toop = el->is_instptr(); + if (toop->instance_klass()->unique_concrete_subklass()) { + // If we load from "AbstractClass[]" we must see "ConcreteSubClass". + const Type* subklass = Type::get_const_type(toop->instance_klass()); + _elemtype = subklass->join_speculative(el); + } + } + } + + if (!arytype->is_loaded()) { + // Only fails for some -Xcomp runs + // The class is unloaded. We have to run this bytecode in the interpreter. + ciKlass* klass = arytype->unloaded_klass(); + + _parse.uncommon_trap(Deoptimization::Reason_unloaded, + Deoptimization::Action_reinterpret, + klass, "!loaded array"); + return true; // top(); + } + + // ary = create_speculative_inline_type_array_checks(ary, arytype, elemtype); + + cast_to_speculative_array_type(arytype); + + if (_parse.needs_range_check(sizetype, _array_index)) { + _parse.create_range_check(_array_index, _array, sizetype); + } else if (_parse.C->log() != nullptr) { + _parse.C->log()->elem("observe that='!need_range_check'"); + } + return false; + } + + ciArrayLoadData* profile_data() const { + if (!UseArrayLoadStoreProfile) { + return nullptr; + } + ciMethodData* md = _parse.method()->method_data(); + if (md == nullptr) { + return nullptr; + } + if (!md->is_mature()) { + return nullptr; + } + ciProfileData* data = md->bci_to_data(_parse.bci()); + if (data == nullptr) { + return nullptr; + } + if (!data->is_ArrayLoadData()) { + return nullptr; + } + return (ciArrayLoadData*) data->as_ArrayLoadData(); + } + + int profiled_not_flat_count() const { + if (profile_data()->not_flat_count() < 0) { + return max_jint; + } + return profile_data()->not_flat_count(); + } + +public: + ArrayLoad(BasicType bt, Parse &parse) : _bt(bt), _parse(parse), _gvn(parse.gvn()), _elemtype((nullptr)), + _array_index(nullptr), _array(nullptr), _region(nullptr), _res_phi(nullptr), + _io_phi(nullptr), _mem_phi(nullptr), _mem(nullptr), _io(nullptr) { + _array_index = _parse.peek(0); // Get from stack without popping + _array = _parse.peek(1); // in case of exception + } + + void pop_stack() { + Node* array_index = _parse.pop(); + Node* array = _parse.pop(); + assert(array_index == _array_index, ""); + assert(array == _array, ""); + } + + // bool emit_load_if_known_flat_array(const TypeOopPtr* element_ptr) { + // if (element_ptr->is_inlinetypeptr()) { + // pop_stack(); + // ciInlineKlass* vk = element_ptr->inline_klass(); + // // Node* flat_array = cast_to_flat_array(array, vk); + // Node* flat_array = _array; + // Node* vt = InlineTypeNode::make_from_flat_array(&_parse, vk, flat_array, _array_index); + // Node* ld = _gvn.transform(vt); + // _parse.push_node(_bt, ld); + // return true; + // } + // return false; + // } + + Node* emit_plain_load(Node* array, bool pin_if_range_check = false, bool safe_for_replace_in_map = false) { + const TypeAryPtr* array_type = _gvn.type(array)->is_aryptr(); + const TypeOopPtr* element_ptr = _elemtype->make_oopptr(); + const TypeAryPtr* adr_type = TypeAryPtr::get_array_body_type(_bt); + DecoratorSet decorator_set = IN_HEAP | IS_ARRAY | C2_CONTROL_DEPENDENT_LOAD; + if (pin_if_range_check && _parse.needs_range_check(array_type->size(), _array_index)) { + // We've emitted a RangeCheck but now insert an additional check between the range check and the actual load. + // We cannot pin the load to two separate nodes. Instead, we pin it conservatively here such that it cannot + // possibly float above the range check at any point. + decorator_set |= C2_UNKNOWN_CONTROL_LOAD; + } + const TypeInt* sizetype = array_type->size(); + if (element_ptr != nullptr && element_ptr->can_be_inline_type() && !array_type->is_null_free()) { + ciArrayLoadData* array_load = profile_data(); + Deoptimization::DeoptReason reason = Deoptimization::Reason_none; + if (array_type->speculative() != nullptr && + !array_type->speculative()->isa_aryptr()->is_not_null_free() && + !_parse.too_many_traps_or_recompiles(Deoptimization::Reason_speculate_class_check)) { + reason = Deoptimization::Reason_speculate_class_check; + } else if (array_load != nullptr && array_load->not_flat_nullable_count() == 0 && array_load-> + not_flat_null_free_count() > 0 && + !_parse.too_many_traps_or_recompiles(Deoptimization::Reason_class_check)) { + reason = Deoptimization::Reason_class_check; + } + if (reason != Deoptimization::Reason_none) { + Node* test = _parse.null_free_array_test(_array, false); + IfNode* iff = _parse.create_and_xform_if(_parse.control(), test, PROB_MIN, COUNT_UNKNOWN); + _parse.set_control(_gvn.transform(new IfTrueNode(iff))); + { + PreserveJVMState pjvms(&_parse); + _parse.uncommon_trap_exact(reason, Deoptimization::Action_maybe_recompile); + } + _parse.set_control(_gvn.transform(new IfFalseNode(iff))); + if (_parse.stopped()) { + if (safe_for_replace_in_map) { + pop_stack(); + } + return _parse.C->top(); + } + const TypeAryPtr* array_type = _gvn.type(_array)->is_aryptr()->cast_to_null_free(true); + array = _gvn.transform(new CheckCastPPNode(_parse.control(), array, array_type)); + if (_parse.needs_range_check(array_type->size(), _array_index)) { + decorator_set |= C2_UNKNOWN_CONTROL_LOAD; + } + } + } + + Node* adr = _parse.array_element_address(array, _array_index, _bt, sizetype, _parse.control()); + assert(adr != _parse.top(), "top should go hand-in-hand with stopped"); + Node* ld = _parse.access_load_at(array, adr, adr_type, _elemtype, _bt, decorator_set); + if (element_ptr != nullptr && element_ptr->is_inlinetypeptr()) { + assert(!array_type->is_null_free() || !element_ptr->maybe_null(), "inline type array elements should never be null"); + ld = InlineTypeNode::make_from_oop(&_parse, ld, element_ptr->inline_klass()); + } + if (safe_for_replace_in_map) { + pop_stack(); + _parse.replace_in_map(_array, array); + } + return ld; + } + + void test_non_flat_array_and_emit_reference_load(float p) { + const TypeAryPtr* array_type = _gvn.type(_array)->is_aryptr(); + Node* test = _parse.flat_array_test(_array, /* flat = */ false); + float this_prob = clamp(p, PROB_MIN, PROB_MAX); + IfNode* iff = _parse.create_and_xform_if(_parse.control(), test, this_prob, COUNT_UNKNOWN); + _parse.set_control(_gvn.transform(new IfTrueNode(iff))); + assert(array_type->is_flat() || _parse.control()->in(0)->as_If()->is_flat_array_check(&_gvn), + "Should be found"); + if (!_parse.stopped()) { + Node *casted_array = _gvn. + transform(new CheckCastPPNode(_parse.control(), _array, array_type->cast_to_not_flat())); + Node *ld = emit_plain_load(casted_array, true); + _res_phi->add_req(ld); + _region->add_req(_parse.control()); + _io_phi->add_req(_parse.i_o()); + _mem_phi->add_req(_parse.reset_memory()); + } + _parse.set_control(_gvn.transform(new IfFalseNode(iff))); + _parse.set_all_memory(_mem); + _parse.set_i_o(_io); + } + + void test_known_flat_array_and_emit_load_flat(ciKlass* klass, float p) { + float this_prob = clamp(p, PROB_MIN, PROB_MAX); + Node* casted_array = nullptr; + Node* next_ctrl = _parse.type_check_receiver(_array, klass, this_prob, &casted_array); + if (_parse.stopped()) { + _parse.set_control(next_ctrl); + _parse.set_all_memory(_mem); + _parse.set_i_o(_io); + return; + } + // TODO: pin load + InlineTypeNode* vt = InlineTypeNode::make_from_flat_array( + &_parse, klass->as_flat_array_klass()->element_klass()->as_inline_klass(), casted_array, _array_index); + Node* null_ctl = _parse.top(); + + _parse.null_check_common(vt->get_null_marker(), T_INT, false, &null_ctl); + + Node* mem = _parse.reset_memory(); + _parse.set_all_memory(mem); + if (!null_ctl->is_top()) { + _res_phi->add_req(_parse.zerocon(T_OBJECT)); + _region->add_req(null_ctl); + _io_phi->add_req(_parse.i_o()); + _mem_phi->add_req(mem); + } + + + Node* ld = _gvn.transform(vt->buffer(&_parse)); + _res_phi->add_req(ld); + _region->add_req(_parse.control()); + _io_phi->add_req(_parse.i_o()); + _mem_phi->add_req(_parse.reset_memory()); + _parse.set_control(next_ctrl); + _parse.set_all_memory(_mem); + _parse.set_i_o(_io); + } + + Node* load_from_unknown_flat_array(const TypeOopPtr* element_ptr) { + Node* ld = nullptr; + + Node* array = _array; + // ciArrayLoadData* array_load = profile_data(); + // if (array_load != nullptr && array_load->null_free_array() && !_parse.too_many_traps_or_recompiles(Deoptimization::Reason_class_check)) { + // Node* test = _parse.null_free_array_test(_array, false); + // IfNode* iff = _parse.create_and_xform_if(_parse.control(), test, PROB_MIN, COUNT_UNKNOWN); + // _parse.set_control(_gvn.transform(new IfTrueNode(iff))); + // { + // PreserveJVMState pjvms(&_parse); + // _parse.uncommon_trap_exact(Deoptimization::Reason_class_check, Deoptimization::Action_maybe_recompile); + // } + // _parse.set_control(_gvn.transform(new IfFalseNode(iff))); + // const TypeAryPtr* array_type = _gvn.type(_array)->is_aryptr()->cast_to_null_free(true); + // array = _gvn.transform(new CheckCastPPNode(_parse.control(), array, array_type)); + // } + + if (element_ptr->is_inlinetypeptr()) { + ciArrayLoadData* array_load = profile_data(); + float null_free_prob = PROB_FAIR; + float null_free_atomic_prob = PROB_FAIR; + if (array_load != nullptr && !_parse.too_many_traps_or_recompiles(Deoptimization::Reason_class_check)) { + int flat_nullable_count = array_load->flat_nullable_count(); + int flat_nullfree_atomic_count = array_load->flat_nullfree_atomic_count(); + int flat_nullfree_not_atomic_count = array_load->flat_nullfree_not_atomic_count(); + ciMegamorphicTypeData* megamorphic_type_data = array_load->megamorphic_type_data(); + for (uint i = 0; i < megamorphic_type_data->row_limit(); ++i) { + ciKlass* receiver = megamorphic_type_data->receiver(i); + if (receiver != nullptr && receiver->is_subtype_of(element_ptr->is_instptr()->instance_klass())) { + ciFlatArrayKlass* flat_array_klass = receiver->as_flat_array_klass();; + if (!flat_array_klass->is_elem_null_free()) { + saturated_add(flat_nullable_count, megamorphic_type_data->receiver_count(i)); + } else { + if (flat_array_klass->is_elem_atomic()) { + saturated_add(flat_nullfree_atomic_count, megamorphic_type_data->receiver_count(i)); + } else { + saturated_add(flat_nullfree_not_atomic_count, megamorphic_type_data->receiver_count(i)); + } + } + } + } + float count = saturated_add(flat_nullable_count, saturated_add(flat_nullfree_atomic_count, flat_nullfree_not_atomic_count)); + if (flat_nullable_count == 0) { + null_free_prob = 1; + } else if (flat_nullable_count == count) { + null_free_prob = 0; + } else { + null_free_prob = clamp(1 - (float)flat_nullable_count / count, PROB_MIN, PROB_MAX); + } + float null_free_count = saturated_add(flat_nullfree_atomic_count, flat_nullfree_not_atomic_count); + if (flat_nullfree_atomic_count == 0) { + null_free_atomic_prob = 0; + } else if (flat_nullfree_atomic_count == null_free_count) { + null_free_atomic_prob = 1; + } else { + null_free_atomic_prob = clamp((float) flat_nullfree_atomic_count / null_free_count, PROB_MIN, PROB_MAX); + } + } + + ciInlineKlass* vk = element_ptr->inline_klass(); + Node* flat_array = _parse.cast_to_flat_array(array, vk); + + InlineTypeNode* vt = InlineTypeNode::make_from_flat_array(&_parse, vk, flat_array, _array_index, null_free_prob, null_free_atomic_prob); + ld = vt; + + if (_region != nullptr && 0) { + Node* null_ctl = _parse.top(); + _parse.null_check_common(vt->get_null_marker(), T_INT, false, &null_ctl); + + _res_phi->add_req(_parse.zerocon(T_OBJECT)); + _region->add_req(null_ctl); + _io_phi->add_req(_parse.i_o()); + _mem_phi->add_req(_parse.reset_memory()); + _parse.set_all_memory(_mem); + } + } else { + ld = _parse.load_from_unknown_flat_array(array, _array_index, element_ptr); + const TypeAryPtr* array_type = _gvn.type(array)->is_aryptr(); + bool is_null_free = array_type->is_null_free() || + (!UseNullableAtomicValueFlattening && !UseNullableNonAtomicValueFlattening); + if (is_null_free) { + ld = _parse.cast_not_null(ld); + } + } + + if (_region != nullptr) { + _region->add_req(_parse.control()); + _res_phi->add_req(ld); + _mem_phi->add_req(_parse.reset_memory()); + _io_phi->add_req(_parse.i_o()); + return nullptr; + } else { + return ld; + } + } + + void create_merge_point() { + _region = new RegionNode(1); + _res_phi = new PhiNode(_region, _elemtype->make_oopptr()); + _io_phi = new PhiNode(_region, Type::ABIO); + _mem = _parse.reset_memory(); + _io = _parse.i_o(); + _parse.set_all_memory(_mem); + _mem_phi = new PhiNode(_region, Type::MEMORY, TypePtr::BOTTOM); + } + + void record_array_profile_for_speculation() { + ciArrayLoadData* array_load = profile_data(); + if (UseTypeSpeculation && array_load != nullptr && array_load->nullable_count() > 0 && array_load->null_free_count() == 0) { + const TypeAryPtr* array_type = _gvn.type(_array)->is_aryptr(); + if (!array_type->is_not_null_free() && array_type->speculative() == nullptr) { + const TypeAryPtr* spec_array_type = TypeAryPtr::BOTTOM->cast_to_not_null_free(); + const TypeOopPtr* spec_type = TypeOopPtr::make(TypePtr::BotPTR, Type::Offset::bottom, TypeOopPtr::InstanceBot, spec_array_type); + Node* cast = new CheckCastPPNode(_parse.control(), _array, array_type->remove_speculative()->join_speculative(spec_type)); + cast = _gvn.transform(cast); + _parse.replace_in_map(_array, cast); + } + } + } + + void finish_merge_point() { + _parse.record_for_igvn(_region); + _parse.set_control(_gvn.transform(_region)); + Node* base = nullptr; + for (uint i = 1; i < _mem_phi->req(); ++i) { + Node* in = _mem_phi->in(i); + Node* new_base = in; + if (in->is_MergeMem()) { + new_base = in->as_MergeMem()->base_memory(); + } + if (base == nullptr) { + base = new_base; + } else if (base != new_base) { + base = NodeSentinel; + } + } + assert(base != nullptr, ""); + if (base != NodeSentinel) { + MergeMemNode* mm = MergeMemNode::make(base); + for (uint i = 1; i < _mem_phi->req(); ++i) { + MergeMemNode* in = _mem_phi->in(i)->isa_MergeMem(); + if (in != nullptr) { + for (MergeMemStream mms(in); mms.next_non_empty(); ) { + if (mms.at_base_memory()) { + continue; + } + Node* mem = mms.memory(); + if (mm->memory_at(mms.alias_idx()) == base) { + Node* phi = PhiNode::make(_region, base, Type::MEMORY, mms.adr_type()); + _gvn.set_type(phi, phi->bottom_type()); + _parse.record_for_igvn(phi); + mm->set_memory_at(mms.alias_idx(), phi); + } + PhiNode* phi = mm->memory_at(mms.alias_idx())->as_Phi(); + phi->set_req(i, mem); + } + } + } + _parse.set_all_memory(_gvn.transform(mm)); + } else { + _parse.record_for_igvn(_mem_phi); + _parse.set_all_memory(_gvn.transform(_mem_phi)); + } + _parse.set_i_o(_gvn.transform(_io_phi)); + Node* ld = _gvn.transform(_res_phi); + ld = _parse.record_profile_for_speculation_at_array_load(ld); + pop_stack(); + _parse.push_node(_bt, ld); + + // record_array_profile_for_speculation(); + } + + void cast_to_speculative_array_type(const TypeAryPtr *&array_type) { + if (!array_type->is_flat() && !array_type->is_not_flat()) { + // For arrays that might be flat, speculate that the array has the exact type reported in the profile data such that + // we can rely on a fixed memory layout (i.e. either a flat layout or not). + Deoptimization::DeoptReason reason = Deoptimization::Reason_speculate_class_check; + ciKlass* speculative_array_type = array_type->speculative_type(); + if (speculative_array_type != nullptr && !_parse.too_many_traps_or_recompiles(reason)) { + // Speculate that this array has the exact type reported by profile data + Node* casted_array = nullptr; + DEBUG_ONLY(Node* old_control = _parse.control();) + Node* slow_ctl = _parse.type_check_receiver(_array, speculative_array_type, 1.0, &casted_array); + if (_parse.stopped()) { + // The check always fails and therefore profile information is incorrect. Don't use it. + assert(old_control == slow_ctl, "type check should have been removed"); + _parse.set_control(slow_ctl); + } else if (!slow_ctl->is_top()) { + { PreserveJVMState pjvms(&_parse); + _parse.set_control(slow_ctl); + _parse.uncommon_trap_exact(reason, Deoptimization::Action_maybe_recompile); + } + _parse.replace_in_map(_array, casted_array); + array_type = _gvn.type(casted_array)->is_aryptr(); + _elemtype = array_type->elem(); + _array = casted_array; + } + } + } + if (!array_type->is_flat() && !array_type->is_not_flat()) { + const TypePtr* speculative = array_type->speculative(); + Deoptimization::DeoptReason reason = Deoptimization::Reason_speculate_class_check; + if (array_type->speculative() != nullptr && + array_type->speculative()->is_aryptr()->is_not_flat() && + !_parse.too_many_traps_or_recompiles(reason)) { + { // Deoptimize if flat array + BuildCutout unless(&_parse, _parse.flat_array_test(_array, /* flat = */ false), PROB_MAX); + _parse.uncommon_trap_exact(reason, Deoptimization::Action_maybe_recompile); + } + assert(!_parse.stopped(), "flat array should have been caught earlier"); + Node* casted_array = _gvn.transform(new CheckCastPPNode(_parse.control(), _array, array_type->cast_to_not_flat())); + _parse.replace_in_map(_array, casted_array); + array_type = _gvn.type(casted_array)->is_aryptr(); + _elemtype = array_type->elem(); + _array = casted_array; + } + } + } + + bool emit() { + emit_null_and_range_checks(); + + // Check for always knowing you are throwing a range-check exception + if (_parse.stopped()) return true; //top(); + + const TypeAryPtr* array_type = _gvn.type(_array)->is_aryptr(); + const TypeOopPtr* element_ptr = _elemtype->make_oopptr(); + + if (array_type->is_not_flat()) { + if (_elemtype == TypeInt::BOOL) { + _bt = T_BOOLEAN; + } + Node* ld = emit_plain_load(_array, false, true); + // pop_stack(); + _parse.push_node(_bt, ld); + ld = _parse.record_profile_for_speculation_at_array_load(ld); + // record_array_profile_for_speculation(); + return true; + } + + if (array_type->is_flat() && element_ptr->is_inlinetypeptr()) { + pop_stack(); + ciInlineKlass* vk = element_ptr->inline_klass(); + Node* flat_array = _array; + Node* vt = InlineTypeNode::make_from_flat_array(&_parse, vk, flat_array, _array_index); + Node* ld = _gvn.transform(vt); + _parse.push_node(_bt, ld); + return true; + } + + if (!array_type->is_not_flat()) { + // Cannot statically determine if array is a flat array, emit runtime check + assert(UseArrayFlattening && is_reference_type(_bt) && element_ptr->can_be_inline_type() && + (!element_ptr->is_inlinetypeptr() || element_ptr->inline_klass()->maybe_flat_in_array()), + "array can't be flat"); + + ciArrayLoadData* array_load = profile_data(); + if (array_load != nullptr) { + int not_flat_count = profiled_not_flat_count(); + Deoptimization::DeoptReason not_flat_reason = Deoptimization::Reason_none; + if (not_flat_count != 0) { + if (array_type->speculative() != nullptr && + array_type->speculative()->is_aryptr()->is_flat()) { + not_flat_count = 0; + not_flat_reason = Deoptimization::Reason_speculate_class_check; + } + } else { + not_flat_reason = Deoptimization::Reason_class_check; + } + ciCallProfile profile = _parse.method()->call_profile_at_bci(_parse.bci()); + int flat_count = profile.count(); + int flat_and_not_flat_count = saturated_add(flat_count, not_flat_count); + if (profile.morphism() > 0 || profile.has_major_receiver()) { + bool not_flat_checked = false; + float prob = 1; + create_merge_point(); + int limit = MAX2(profile.morphism(), 1); + bool done = false; + for (int i = 0; i < limit || !not_flat_checked;) { + assert(!_parse.stopped(), ""); + int count = i < limit ? profile.receiver_count(i) : not_flat_count; + if (not_flat_count >= count && !not_flat_checked) { + not_flat_checked = true; + if (profile.morphism() > 0 && not_flat_count == 0 && !_parse.too_many_traps_or_recompiles(not_flat_reason)) { + PreserveJVMState pjvms(&_parse); + _parse.uncommon_trap_exact(not_flat_reason, Deoptimization::Action_maybe_recompile); + done = true; + } else if (profile.morphism() <= 0 && not_flat_count == 0 && !_parse.too_many_traps_or_recompiles(not_flat_reason)) { + const TypeAryPtr* array_type = _gvn.type(_array)->is_aryptr(); + Node* test = _parse.flat_array_test(_array, /* flat = */ false); + IfNode* iff = _parse.create_and_xform_if(_parse.control(), test, PROB_MIN, COUNT_UNKNOWN); + _parse.set_control(_gvn.transform(new IfTrueNode(iff))); + assert(array_type->is_flat() || _parse.control()->in(0)->as_If()->is_flat_array_check(&_gvn), + "Should be found"); + { + PreserveJVMState pjvms(&_parse); + _parse.uncommon_trap_exact(not_flat_reason, Deoptimization::Action_maybe_recompile); + } + _parse.set_control(_gvn.transform(new IfFalseNode(iff))); + load_from_unknown_flat_array(element_ptr); + done = true; + } else { + float p = ((float) not_flat_count) / ((float) flat_and_not_flat_count); + test_non_flat_array_and_emit_reference_load(p / prob); + prob = 1 - p; + } + } else { + float p = ((float) profile.receiver_count(i)) / ((float) flat_and_not_flat_count); + ciKlass* klass = profile.receiver(i); + test_known_flat_array_and_emit_load_flat(klass, p / prob); + i++; + prob = 1 - p; + } + } + if (!done) { + if (!_parse.too_many_traps_or_recompiles(Deoptimization::Reason_class_check)) { + PreserveJVMState pjvms(&_parse); + _parse.uncommon_trap_exact(Deoptimization::Reason_class_check, Deoptimization::Action_maybe_recompile); + } else { + load_from_unknown_flat_array(element_ptr); + } + } + finish_merge_point(); + return true; + } + if (flat_count == 0 && not_flat_count > 0 && !_parse.too_many_traps_or_recompiles(Deoptimization::Reason_class_check)) { + { + // Deoptimize if flat array + BuildCutout unless(&_parse, _parse.flat_array_test(_array, /* flat = */ false), PROB_MAX); + _parse.uncommon_trap_exact(Deoptimization::Reason_class_check, Deoptimization::Action_maybe_recompile); + } + assert(!_parse.stopped(), "flat array should have been caught earlier"); + Node* casted_array = _gvn.transform( + new CheckCastPPNode(_parse.control(), _array, array_type->cast_to_not_flat())); + _parse.replace_in_map(_array, casted_array); + _array = casted_array; + Node* ld = emit_plain_load(casted_array, true); + ld = _parse.record_profile_for_speculation_at_array_load(ld); + pop_stack(); + _parse.push_node(_bt, ld); + record_array_profile_for_speculation(); + return true; + } + if (flat_count != 0 && not_flat_count == 0 && !_parse.too_many_traps_or_recompiles(not_flat_reason)) { + { + // Deoptimize if not flat array + BuildCutout unless(&_parse, _parse.flat_array_test(_array, /* flat = */ true), PROB_MAX); + _parse.uncommon_trap_exact(not_flat_reason, Deoptimization::Action_maybe_recompile); + } + assert(!_parse.stopped(), "non flat array should have been caught earlier"); + Node* ld = load_from_unknown_flat_array(element_ptr); + pop_stack(); + _parse.push_node(_bt, ld); + // record_array_profile_for_speculation(); + return true; + } + if (flat_count != 0 && not_flat_count != 0) { + create_merge_point(); + float p = ((float) not_flat_count) / ((float) flat_and_not_flat_count); + test_non_flat_array_and_emit_reference_load(p); + load_from_unknown_flat_array(element_ptr); + finish_merge_point(); + return true; + } + } + create_merge_point(); + test_non_flat_array_and_emit_reference_load(PROB_FAIR); + load_from_unknown_flat_array(element_ptr); + finish_merge_point(); + return true; + } + return false; + } +}; //---------------------------------array_load---------------------------------- void Parse::array_load(BasicType bt) { + if (!UseNewCode) { + ArrayLoad array_load(bt, *this); + if (array_load.emit()) { + return; + } + + ShouldNotReachHere(); + } const Type* elemtype = Type::TOP; Node* prep_array = prepare_array_addressing(bt, 0, elemtype); if (stopped()) return; // guaranteed null or range check @@ -90,6 +711,144 @@ void Parse::array_load(BasicType bt) { // Cannot statically determine if array is a flat array, emit runtime check assert(UseArrayFlattening && is_reference_type(bt) && element_ptr->can_be_inline_type() && (!element_ptr->is_inlinetypeptr() || element_ptr->inline_klass()->maybe_flat_in_array()), "array can't be flat"); + // + // if (element_ptr->is_inlinetypeptr()) { + // ciInlineKlass* vk = element_ptr->inline_klass(); + // // Node* flat_array = cast_to_flat_array(array, vk); + // Node* flat_array = array; + // Node* vt = InlineTypeNode::make_from_flat_array(this, vk, flat_array, array_index); + // Node* ld = _gvn.transform(vt); + // push_node(bt, ld); + // return; + // } + // ciMethodData* md = method()->method_data(); + // if (md != nullptr && md->is_mature()) { + // ciProfileData* data = md->bci_to_data(bci()); + // if (data != nullptr && data->is_ArrayLoadData()) { + // ciArrayLoadData* array_load = (ciArrayLoadData*) data->as_ArrayLoadData(); + // int not_flat_count = array_load->not_flat_count(); + // if (not_flat_count < 0) { + // not_flat_count = max_jint; + // } + // ciCallProfile profile = method()->call_profile_at_bci(bci()); + // int flat_count = profile.count(); + // int flat_and_not_flat_count = saturated_add(flat_count, not_flat_count); + // if (profile.morphism() > 0) { + // bool not_flat_checked = false; + // float prob = 1; + // Node* region = new RegionNode(profile.morphism() * 2 + 3); + // Node* res_phi = new PhiNode(region, TypeOopPtr::BOTTOM); + // Node* io_phi = new PhiNode(region, Type::ABIO); + // Node* mem = reset_memory(); + // Node* io = i_o(); + // set_all_memory(mem); + // Node* mem_phi = new PhiNode(region, Type::MEMORY, TypePtr::BOTTOM); + // int j = 1; + // for (int i = 0; i < profile.morphism() || !not_flat_checked; ) { + // int count = i < profile.morphism() ? profile.receiver_count(i) : not_flat_count; + // if (not_flat_count >= count && !not_flat_checked) { + // not_flat_checked = true; + // if (not_flat_count == 0 && !too_many_traps_or_recompiles(Deoptimization::Reason_class_check)) { + // PreserveJVMState pjvms(this); + // inc_sp(2); + // uncommon_trap_exact(Deoptimization::Reason_class_check, Deoptimization::Action_maybe_recompile); + // } else { + // Node* test = flat_array_test(array, /* flat = */ false); + // float p = ((float)not_flat_count) / ((float)flat_and_not_flat_count); + // float this_prob = clamp(p / prob, PROB_MIN, PROB_MAX); + // IfNode* iff = create_and_xform_if(control(), test, this_prob, COUNT_UNKNOWN); + // set_control(_gvn.transform(new IfTrueNode(iff))); + // assert(array_type->is_flat() || control()->in(0)->as_If()->is_flat_array_check(&_gvn), "Should be found"); + // const TypeAryPtr* adr_type = TypeAryPtr::get_array_body_type(bt); + // DecoratorSet decorator_set = IN_HEAP | IS_ARRAY | C2_CONTROL_DEPENDENT_LOAD; + // if (needs_range_check(array_type->size(), array_index)) { + // // We've emitted a RangeCheck but now insert an additional check between the range check and the actual load. + // // We cannot pin the load to two separate nodes. Instead, we pin it conservatively here such that it cannot + // // possibly float above the range check at any point. + // decorator_set |= C2_UNKNOWN_CONTROL_LOAD; + // } + // Node* ld = access_load_at(array, adr, adr_type, element_ptr, bt, decorator_set); + // if (element_ptr->is_inlinetypeptr()) { + // ld = InlineTypeNode::make_from_oop(this, ld, element_ptr->inline_klass()); + // } + // res_phi->init_req(j, _gvn.transform(ld)); + // region->init_req(j, control()); + // io_phi->init_req(j, i_o()); + // set_control(_gvn.transform(new IfFalseNode(iff))); + // mem_phi->init_req(j, reset_memory()); + // set_all_memory(mem); + // set_i_o(io); + // prob = 1 - p; + // } + // } else { + // inc_sp(2); + // float p = ((float)profile.receiver_count(i)) / ((float)flat_and_not_flat_count); + // float this_prob = clamp(p / prob, PROB_MIN, PROB_MAX); + // Node* casted_array = nullptr; + // ciKlass* klass = profile.receiver(i); + // Node* next_ctrl = type_check_receiver(array, klass, this_prob, &casted_array); + // InlineTypeNode* vt = InlineTypeNode::make_from_flat_array(this, klass->as_flat_array_klass()->element_klass()->as_inline_klass(), casted_array, array_index); + // Node* null_ctl = top(); + // + // null_check_common(vt->get_null_marker(), T_INT, false, &null_ctl); + // + // res_phi->init_req(j, zerocon(T_OBJECT)); + // region->init_req(j, null_ctl); + // io_phi->init_req(j, i_o()); + // mem_phi->init_req(j, reset_memory()); + // set_all_memory(mem); + // + // j++; + // + // Node* ld = _gvn.transform(vt->buffer(this)); + // res_phi->init_req(j, ld); + // region->init_req(j, control()); + // io_phi->init_req(j, i_o()); + // set_control(next_ctrl); + // mem_phi->init_req(j, reset_memory()); + // set_all_memory(mem); + // set_i_o(io); + // prob = 1 - p; + // i++; + // dec_sp(2); + // } + // j++; + // } + // if (not_flat_count != 0 && !too_many_traps_or_recompiles(Deoptimization::Reason_class_check)) { + // PreserveJVMState pjvms(this); + // inc_sp(2); + // uncommon_trap_exact(Deoptimization::Reason_class_check, Deoptimization::Action_maybe_recompile); + // } else if (too_many_traps_or_recompiles(Deoptimization::Reason_class_check)) { + // Node* unknown_inline_type = load_from_unknown_flat_array(array, array_index, element_ptr); + // + // region->init_req(j, control()); + // res_phi->init_req(j, unknown_inline_type); + // mem_phi->init_req(j, reset_memory()); + // io_phi->init_req(j, i_o()); + // } + // set_control(_gvn.transform(region)); + // set_all_memory(_gvn.transform(mem_phi)); + // set_i_o(_gvn.transform(io_phi)); + // Node* ld = _gvn.transform(res_phi); + // ld = record_profile_for_speculation_at_array_load(ld); + // push_node(bt, ld); + // return; + // } + // if (flat_count == 0 && not_flat_count > 0 && !too_many_traps_or_recompiles(Deoptimization::Reason_class_check)) { + // { // Deoptimize if flat array + // BuildCutout unless(this, flat_array_test(array, /* flat = */ false), PROB_MAX); + // inc_sp(2); + // uncommon_trap_exact(Deoptimization::Reason_class_check, Deoptimization::Action_maybe_recompile); + // } + // assert(!stopped(), "flat array should have been caught earlier"); + // Node* casted_array = _gvn.transform(new CheckCastPPNode(control(), array, array_type->cast_to_not_flat())); + // replace_in_map(array, casted_array); + // } else if (profile.has_major_receiver()) { + // + // } + // } + // } + // IdealKit ideal(this); IdealVariable res(ideal); ideal.declarations_done(); @@ -175,7 +934,7 @@ Node* Parse::load_from_unknown_flat_array(Node* array, Node* array_index, const PreserveReexecuteState preexecs(this); jvms()->set_bci(_bci); jvms()->set_should_reexecute(true); - inc_sp(2); + // inc_sp(2); kill_dead_locals(); call = make_runtime_call(RC_NO_LEAF | RC_NO_IO, OptoRuntime::load_unknown_inline_Type(), diff --git a/src/hotspot/share/opto/type.cpp b/src/hotspot/share/opto/type.cpp index f7542b333e0..30cfdc2a0b1 100644 --- a/src/hotspot/share/opto/type.cpp +++ b/src/hotspot/share/opto/type.cpp @@ -5175,9 +5175,17 @@ const TypeAryPtr* TypeAryPtr::cast_to_null_free(bool null_free) const { } else { new_elem = new_elem->meet_speculative(TypePtr::NULL_PTR); } + const TypePtr* speculative = _speculative; + if (speculative != nullptr && speculative->isa_aryptr()) { + if (null_free && speculative->is_aryptr()->is_not_null_free()) { + speculative = nullptr; + } else { + speculative = speculative->is_aryptr()->cast_to_null_free(null_free); + } + } new_elem = elem->isa_narrowoop() ? new_elem->make_narrowoop() : new_elem; const TypeAry* new_ary = TypeAry::make(new_elem, size(), is_stable(), is_flat(), is_not_flat(), is_not_null_free(), is_atomic()); - const TypeAryPtr* res = make(ptr(), const_oop(), new_ary, klass(), klass_is_exact(), _offset, _field_offset, _instance_id, _speculative, _inline_depth, _is_autobox_cache); + const TypeAryPtr* res = make(ptr(), const_oop(), new_ary, klass(), klass_is_exact(), _offset, _field_offset, _instance_id, speculative, _inline_depth, _is_autobox_cache); if (res->speculative() == res->remove_speculative()) { return res->remove_speculative(); } diff --git a/test/hotspot/jtreg/compiler/lib/ir_framework/IRNode.java b/test/hotspot/jtreg/compiler/lib/ir_framework/IRNode.java index 950bec4c914..2e197a6e8b3 100644 --- a/test/hotspot/jtreg/compiler/lib/ir_framework/IRNode.java +++ b/test/hotspot/jtreg/compiler/lib/ir_framework/IRNode.java @@ -3291,6 +3291,11 @@ public static void anyStoreOfNodes(String irNodePlaceholder, String fieldHolder) beforeMatchingNameRegex(OPAQUE_CONSTANT_BOOL, "OpaqueConstantBool"); } + public static final String BIMORPHIC_TRAP = PREFIX + "BIMORPHIC_TRAP" + POSTFIX; + static { + trapNodes(BIMORPHIC_TRAP, "bimorphic"); + } + /* * Utility methods to set up IR_NODE_MAPPINGS. */ diff --git a/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java b/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java new file mode 100644 index 00000000000..56f41121bc9 --- /dev/null +++ b/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java @@ -0,0 +1,1041 @@ +/* + * Copyright (c) 2026 IBM Corporation. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +/** + * @test + * @library /test/lib / + * @requires vm.flagless + * @enablePreview + * @modules java.base/jdk.internal.value + * @modules java.base/jdk.internal.vm.annotation + * @requires (os.simpleArch == "x64" | os.simpleArch == "aarch64") + * @library /test/lib / + * @build jdk.test.whitebox.WhiteBox + * @run driver jdk.test.lib.helpers.ClassFileInstaller jdk.test.whitebox.WhiteBox + * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI ${test.main.class} + */ + +package compiler.valhalla.inlinetypes; + +import compiler.lib.ir_framework.*; +import jdk.internal.value.ValueClass; +import jdk.internal.vm.annotation.LooselyConsistentValue; +import jdk.test.whitebox.WhiteBox; +import compiler.whitebox.CompilerWhiteBoxTest; +import java.lang.reflect.Method; + +public class TestArrayLoadProfiling { + private final static WhiteBox WHITE_BOX = WhiteBox.getWhiteBox(); + + public static void main(String[] args) { + TestFramework.runWithFlags("--enable-preview", "--add-exports", "java.base/jdk.internal.value=ALL-UNNAMED", "-Xbootclasspath/a:.", "-XX:+UnlockDiagnosticVMOptions", "-XX:+WhiteBoxAPI", "-XX:-TieredCompilation"); + TestFramework.runWithFlags("--enable-preview", "--add-exports", "java.base/jdk.internal.value=ALL-UNNAMED", "-Xbootclasspath/a:.", "-XX:+UnlockDiagnosticVMOptions", "-XX:+WhiteBoxAPI", "-XX:-ProfileInterpreter"); + TestFramework.runWithFlags("--enable-preview", "--add-exports", "java.base/jdk.internal.value=ALL-UNNAMED", "-Xbootclasspath/a:.", "-XX:+UnlockDiagnosticVMOptions", "-XX:+WhiteBoxAPI"); + } + + static final MyValue1[] array1 = { new MyValue1((byte)42) }; + static MyValue2[] array2 = { new MyValue2((byte)42) }; + static MyValue1[] array3 = (MyValue1[])ValueClass.newNullRestrictedNonAtomicArray(MyValue1.class, 1, new MyValue1((byte)42)); + static MyValue2[] array4 = (MyValue2[])ValueClass.newNullRestrictedNonAtomicArray(MyValue2.class, 1, new MyValue2((byte)42)); + static A[] array5 = { new A() }; + static MyValue1[] array6 = (MyValue1[])ValueClass.newReferenceArray(MyValue1.class, 1); + static MyValue2[] array7 = (MyValue2[])ValueClass.newReferenceArray(MyValue2.class, 1); + static MyValue1[] array8 = (MyValue1[])ValueClass.newNullRestrictedAtomicArray(MyValue1.class, 1, new MyValue1((byte)42)); + static MyValue1[] array9 = (MyValue1[])ValueClass.newNullableAtomicArray(MyValue1.class, 1); + static MyValue3[] array10 = (MyValue3[])ValueClass.newNullRestrictedNonAtomicArray(MyValue3.class, 1, new MyValue3(42)); + static MyValue3[] array11 = (MyValue3[])ValueClass.newNullRestrictedAtomicArray(MyValue3.class, 1, new MyValue3(42)); + static MyValue3[] array12 = (MyValue3[])ValueClass.newReferenceArray(MyValue3.class, 1); + static MyValue4[] array13 = (MyValue4[])ValueClass.newNullRestrictedNonAtomicArray(MyValue4.class, 1, new MyValue4((byte)42)); // atomic + static MyValue4[] array14 = (MyValue4[])ValueClass.newNullRestrictedAtomicArray(MyValue4.class, 1, new MyValue4((byte)42)); // atomic + static MyValue4[] array15 = (MyValue4[])ValueClass.newNullableAtomicArray(MyValue4.class, 1); + static MyValue4[] array16 = { new MyValue4((byte)42) }; + static MyValue5[] array17 = (MyValue5[])ValueClass.newNullRestrictedNonAtomicArray(MyValue5.class, 1, new MyValue5((byte)42)); // non atomic + static MyValue5[] array18 = (MyValue5[])ValueClass.newNullRestrictedAtomicArray(MyValue5.class, 1, new MyValue5((byte)42)); // non atomic + static MyValue5[] array19 = (MyValue5[])ValueClass.newNullableAtomicArray(MyValue5.class, 1); + static MyValue5[] array20 = { new MyValue5((byte)42) }; + static MyValue4[] array21 = (MyValue4[])ValueClass.newReferenceArray(MyValue4.class, 1); + static MyValue2[] array22 = (MyValue2[])ValueClass.newNullRestrictedAtomicArray(MyValue2.class, 1, new MyValue2((byte)42)); + static A[][] array23 = {{ new A() }}; + static MyValue1[][] array24 = {{ new MyValue1((byte)42) }}; + static B[][] array25 = {{ new B() }}; + static MyValue2[][] array26 = {{ new MyValue2((byte)42) }}; + static MyValue3[][] array27 = { (MyValue3[])ValueClass.newNullRestrictedAtomicArray(MyValue3.class, 1, new MyValue3(42)) }; + static MyValue6[][] array28 = { (MyValue6[])ValueClass.newNullRestrictedAtomicArray(MyValue6.class, 1, new MyValue6(42)) }; + static { + array6[0] = new MyValue1((byte)42); + array7[0] = new MyValue2((byte)42); + array8[0] = new MyValue1((byte)42); + array9[0] = new MyValue1((byte)42); + array12[0] = new MyValue3((byte)42); + array15[0] = new MyValue4((byte)42); + array19[0] = new MyValue5((byte)42); + array21[0] = new MyValue4((byte)42); + } + + static I staticField; + + @Test + @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "1", IRNode.TRAP, "4", IRNode.CALL, "4", IRNode.IF, "4" }) + @IR(failOn = IRNode.ALLOC) + public static void test1(I[] array) { + test1Inline(array[0]); + } + + @Run(test = "test1") + @Warmup(10_000) + public static void test1Runner() { + test1(array1); + test1Inline(array2[0]); + test1Inline(array3[0]); + test1Inline(array4[0]); + test1Inline(array5[0]); + } + + @ForceInline + static void test1Inline(I i) { + i.m(); + } + + static void trapAndRecompile(String name, Runnable causesTrap) throws Exception { + Method m = TestArrayLoadProfiling.class.getDeclaredMethod(name, I[].class); + if (TestFramework.isStableDeopt(m, CompLevel.C2) && (!WHITE_BOX.isMethodCompiled(m) || WHITE_BOX.getMethodCompilationLevel(m) != CompilerWhiteBoxTest.COMP_LEVEL_FULL_OPTIMIZATION)) { + throw new RuntimeException("should be compiled"); + } + int i = 0; + do { + causesTrap.run(); + i++; + if (i > 10) { + if (TestFramework.isStableDeopt(m, CompLevel.C2)) { + throw new RuntimeException("should not be compiled anymore"); + } else { + break; + } + } + } while (WHITE_BOX.isMethodCompiled(m)); + WHITE_BOX.enqueueMethodForCompilation(m, CompilerWhiteBoxTest.COMP_LEVEL_FULL_OPTIMIZATION); + if (TestFramework.isStableDeopt(m, CompLevel.C2) && (!WHITE_BOX.isMethodCompiled(m) || WHITE_BOX.getMethodCompilationLevel(m) != CompilerWhiteBoxTest.COMP_LEVEL_FULL_OPTIMIZATION)) { + throw new RuntimeException("should be compiled"); + } + } + + @Test + @IR(counts = { IRNode.NULL_CHECK_TRAP, "1", IRNode.RANGE_CHECK_TRAP, "1", IRNode.DYNAMIC_CALL_OF_METHOD, "m", "1", IRNode.UNHANDLED_TRAP, "2", IRNode.CALL, "6", IRNode.ALLOC, "2" }, + phase = { CompilePhase.BEFORE_MACRO_EXPANSION }, applyIf = { "ProfileInterpreter", "true"}) + @IR(failOn = { IRNode.CLASS_CHECK_TRAP } , applyIf = { "ProfileInterpreter", "true" }) + public static void test2(I[] array) { + test2Inline(array[0]); + } + + @Run(test = "test2") + @Warmup(10_000) + public static void test2Runner(RunInfo info) throws Exception { + if (info.isWarmUp()) { + test2(array1); + test2Inline(array2[0]); + test2Inline(array3[0]); + test2Inline(array4[0]); + test2Inline(array5[0]); + } else { + trapAndRecompile("test2", () -> { test2(array2); }); + } + } + + @ForceInline + static void test2Inline(I i) { + i.m(); + } + + @Test + @IR(counts = { IRNode.NULL_CHECK_TRAP, "1", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "1", IRNode.TRAP, "3", IRNode.CALL, "3", IRNode.IF, "4" }) + @IR(failOn = IRNode.ALLOC) + public static void test3(I[] array) { + test3Inline(array[0]); + } + + @Run(test = "test3") + @Warmup(10_000) + public static void test3Runner() { + test3(array3); + test3(array4); + } + + @ForceInline + static void test3Inline(I i) { + i.m(); + } + + @Test + @IR(failOn = { IRNode.CLASS_CHECK_TRAP } , applyIf = { "ProfileInterpreter", "true" }) + public static void test4(I[] array) { + test4Inline(array[0]); + } + + @Run(test = "test4") + @Warmup(10_000) + public static void test4Runner(RunInfo info) throws Exception { + if (info.isWarmUp()) { + test4(array3); + test4(array4); + } else { + trapAndRecompile("test4", () -> { test4(array1); }); + } + } + + @ForceInline + static void test4Inline(I i) { + i.m(); + } + + @Test + @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "1", IRNode.TRAP, "4", IRNode.CALL, "4", IRNode.IF, "6" }) + @IR(failOn = IRNode.ALLOC) + public static void test5(I[] array) { + test5Inline(array[0]); + } + + @Run(test = "test5") + @Warmup(10_000) + public static void test5Runner() { + test5(array1); + test5(array2); + } + + @ForceInline + static void test5Inline(I i) { + i.m(); + } + + @Test + @IR(failOn = { IRNode.CLASS_CHECK_TRAP } , applyIf = { "ProfileInterpreter", "true" }) + public static void test6(I[] array) { + test6Inline(array[0]); + } + + @Run(test = "test6") + @Warmup(10_000) + public static void test6Runner(RunInfo info) throws Exception { + if (info.isWarmUp()) { + test6(array1); + test6(array2); + } else { + trapAndRecompile("test6", () -> { test6(array3); }); + } + } + + @ForceInline + static void test6Inline(I i) { + i.m(); + } + + @Test + @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "2", IRNode.TRAP, "5", IRNode.CALL, "5", IRNode.IF, "6" }) + @IR(failOn = IRNode.ALLOC) + public static void test7(I[] array) { + test7Inline(array[0]); + } + + @Run(test = "test7") + @Warmup(10_000) + public static void test7Runner() { + test7Inline(array1[0]); + test7Inline(array2[0]); + test7Inline(array3[0]); + test7Inline(array4[0]); + test7(array5); + } + + @ForceInline + static void test7Inline(I i) { + i.m(); + } + + @Test + @IR(failOn = { IRNode.CLASS_CHECK_TRAP } , applyIf = { "ProfileInterpreter", "true" }) + public static void test8(I[] array) { + test8Inline(array[0]); + } + + @Run(test = "test8") + @Warmup(10_000) + public static void test8Runner(RunInfo info) throws Exception { + if (info.isWarmUp()) { + test8Inline(array1[0]); + test8Inline(array2[0]); + test8Inline(array3[0]); + test8Inline(array4[0]); + test8(array5); + } else { + trapAndRecompile("test8", () -> { test8(array1); }); + } + } + + @ForceInline + static void test8Inline(I i) { + i.m(); + } + + // if (array == null) { + // trap1; + // } + // if (0 not in range of array) { + // trap2; + // } + // if (array flat) { + // if (array.klass == MyValue1[]) { + // if (array[0] == null) { + // trap3; + // } + // // inlined call + // } else if (array.klass == MyValue2[]) { + // if (array[0] == null) { + // trap3; + // } + // // inlined call + // } else { + // trap4; + // } + // } else { + // if (array[0] == null) { + // trap3; + // } + // if (array[0].klass == MyValue1) { + // // inlined call + // } else if (array[0].klass == MyValue2) { + // // inlined call + // } else { + // trap5; + // } + // } + @Test + @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "1", IRNode.BIMORPHIC_TRAP, "1", IRNode.TRAP, "5", IRNode.CALL, "5", IRNode.IF, "11" }) + @IR(failOn = IRNode.ALLOC) + public static void test9(I[] array) { + test9Inline(array[0]); + } + + @Run(test = "test9") + @Warmup(10_000) + public static void test9Runner() { + test9(array1); + test9(array2); + test9(array6); + test9(array7); + } + + @ForceInline + static void test9Inline(I i) { + i.m(); + } + + @Test + @IR(failOn = { IRNode.CLASS_CHECK_TRAP } , applyIf = { "ProfileInterpreter", "true" }) + public static void test10(I[] array) { + test10Inline(array[0]); + } + + @Run(test = "test10") + @Warmup(10_000) + public static void test10Runner(RunInfo info) throws Exception { + if (info.isWarmUp()) { + test10(array1); + test10(array2); + test10(array6); + test10(array7); + } else { + trapAndRecompile("test10", () -> { test10(array3); }); + } + } + + @ForceInline + static void test10Inline(I i) { + i.m(); + } + + // if (array == null) { + // trap1; + // } + // if (0 not in range of array) { + // trap2; + // } + // if (array.klass == MyValue1[]) { + // if (array[0] == null) { + // trap3; + // } + // // inlined call + // } else { + // if (array flat) { + // elt = load_unknown_inline(); + // if (elt == null) { + // trap3; + // } + // if (elt.klass == MyValue1) { + // // inlined call + // } else if (elt.klass == MyValue2) { + // // inlined call + // } else { + // trap4; + // } + // } else { + // trap5; + // } + // } + @Test + @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "1", IRNode.BIMORPHIC_TRAP, "1", IRNode.TRAP, "5", IRNode.CALL, "6", IRNode.IF, "9" }) + @IR(failOn = IRNode.ALLOC) + public static void test11(I[] array) { + test11Inline(array[0]); + } + + @Run(test = "test11") + @Warmup(10_000) + public static void test11Runner() { + for (int i = 0; i < 50; i++) { + test11(array1); + } + test11(array2); + test11(array3); + test11(array4); + } + + @ForceInline + static void test11Inline(I i) { + i.m(); + } + + @Test + @IR(failOn = { IRNode.CLASS_CHECK_TRAP } , applyIf = { "ProfileInterpreter", "true" }) + public static void test12(I[] array) { + test12Inline(array[0]); + } + + @Run(test = "test12") + @Warmup(10_000) + public static void test12Runner(RunInfo info) throws Exception { + if (info.isWarmUp()) { + for (int i = 0; i < 50; i++) { + test12(array1); + } + test12(array2); + test12(array3); + test12(array4); + } else { + trapAndRecompile("test12", () -> { test12(array5); }); + } + } + + @ForceInline + static void test12Inline(I i) { + i.m(); + } + + @Test + @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "1", IRNode.BIMORPHIC_TRAP, "1", IRNode.TRAP, "5", IRNode.CALL, "6", IRNode.IF, "7" }) + @IR(failOn = IRNode.ALLOC) + public static void test13(I[] array) { + test13Inline(array[0]); + } + + @Run(test = "test13") + @Warmup(10_000) + public static void test13Runner() { + test13(array1); + test13(array2); + test13(array3); + test13(array4); + } + + @ForceInline + static void test13Inline(I i) { + i.m(); + } + + @Test + @IR(failOn = { IRNode.CLASS_CHECK_TRAP } , applyIf = { "ProfileInterpreter", "true" }) + public static void test14(I[] array) { + test14Inline(array[0]); + } + + @Run(test = "test14") + @Warmup(10_000) + public static void test14Runner(RunInfo info) throws Exception { + if (info.isWarmUp()) { + test14(array1); + test14(array2); + test14(array3); + test14(array4); + } else { + trapAndRecompile("test14", () -> { test14(array5); }); + } + } + + @ForceInline + static void test14Inline(I i) { + i.m(); + } + + @Test + @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.BIMORPHIC_TRAP, "1", IRNode.TRAP, "4", IRNode.CALL, "5", IRNode.IF, "7" }) + @IR(failOn = IRNode.ALLOC) + public static void test15(I[] array) { + test15Inline(array[0]); + } + + @Run(test = "test15") + @Warmup(10_000) + public static void test15Runner() { + test15(array1); + test15(array2); + test15(array3); + test15(array4); + test15(array6); + test15(array7); + } + + @ForceInline + static void test15Inline(I i) { + i.m(); + } + + @Test + @IR(failOn = { IRNode.CLASS_CHECK_TRAP, IRNode.BIMORPHIC_TRAP } ) + public static void test16(I[] array) { + test16Inline(array[0]); + } + + @Run(test = "test16") + @Warmup(10_000) + public static void test16Runner(RunInfo info) throws Exception { + if (info.isWarmUp()) { + test16(array1); + test16(array2); + test16(array3); + test16(array4); + test16(array6); + test16(array7); + } else { + trapAndRecompile("test16", () -> { test16(array5); }); + } + } + + @ForceInline + static void test16Inline(I i) { + i.m(); + } + + @Test + @IR(counts = { IRNode.NULL_CHECK_TRAP, "1", IRNode.RANGE_CHECK_TRAP, "1", IRNode.TRAP, "2", IRNode.CALL, "4", IRNode.IF, "4" }) + @IR(failOn = IRNode.ALLOC) + public static void test17(I[] array) { + test17Inline(array[0]); + } + + @Run(test = "test17") + @Warmup(0) + public static void test17Runner() { + test17(array1); + } + + @ForceInline + static void test17Inline(I i) { + i.m(); + } + + @Test + @IR(counts = { IRNode.NULL_CHECK_TRAP, "1", IRNode.TRAP, "1", IRNode.CALL, "1", IRNode.IF, "1" }) + @IR(failOn = IRNode.ALLOC) + public static void test18() { + array1[0].m(); + } + + @Test + @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.TRAP, "3", IRNode.CALL, "3", IRNode.IF, "3" }) + @IR(failOn = IRNode.ALLOC) + public static void test19(A[] array) { + array[0].m(); + } + + @Run(test = "test19") + @Warmup(10_000) + public static void test19Runner() { + test19(array5); + } + + @Test + @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.TRAP, "3", IRNode.CALL, "3", IRNode.IF, "8" }) + @IR(failOn = IRNode.ALLOC) + public static void test20(MyValue1[] array) { + array[0].m(); + } + + @Run(test = "test20") + @Warmup(0) + public static void test20Runner() { + test20(array1); + } + + @Test + @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "1", IRNode.TRAP, "4", IRNode.CALL, "4", IRNode.IF, "7" }) + @IR(failOn = IRNode.ALLOC) + public static void test21() { + I[] array = array16; + test21Inline(array); + } + + @Run(test = "test21") + @Warmup(10_000) + public static void test21Runner() { + test21(); + test21Inline(array2); // flat, nullable + test21Inline(array20); // flat, nullable + test21Inline(array5); // not flat + test21Inline(array1); // flat, nullable + } + + @ForceInline + static void test21Inline(I[] array) { + array[0].m(); + } + + @Test + @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "2", IRNode.TRAP, "5", IRNode.CALL, "5", IRNode.IF, "6" }) + @IR(failOn = IRNode.ALLOC) + public static void test22() { + I[] array = array16; + test22Inline(array); + } + + @Run(test = "test22") + @Warmup(10_000) + public static void test22Runner() { + test22(); + test22Inline(array2); // flat, nullable + test22Inline(array20); // flat, nullable + test22Inline(array1); // flat, nullable + } + + @ForceInline + static void test22Inline(I[] array) { + array[0].m(); + } + + // @Test + // @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "1", IRNode.TRAP, "4", IRNode.CALL, "4", IRNode.IF, "9" }) + // @IR(failOn = IRNode.ALLOC) + // public static void test22(I[] array) { + // test22Inline(array[0]); + // } + + // @Run(test = "test22") + // public static void test22Runner() { + // test22(array1); + // test22(array3); + // test22(array6); + // test22(array8); + // test22(array9); + // test22Inline(array2[0]); + // test22Inline(array4[0]); + // test22Inline(array5[0]); + // } + + // @ForceInline + // static void test22Inline(I i) { + // i.m(); + // } + + @Test + @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "2", IRNode.TRAP, "5", IRNode.CALL, "5", IRNode.IF, "9" }) + @IR(failOn = IRNode.ALLOC) + public static void test23(I[] array) { + test23Inline(array[0]); + } + + @Run(test = "test23") + @Warmup(10_000) + public static void test23Runner() { + test23(array1); // flat nullable + //test23(array3); // flat null free non atomic + test23(array6); // not flat + test23(array8); // flat null free atomic + test23(array9); // flat nullable atomic + test23Inline(array2[0]); + test23Inline(array4[0]); + test23Inline(array5[0]); + } + + @ForceInline + static void test23Inline(I i) { + i.m(); + } + + @Test + @IR(counts = { IRNode.NULL_CHECK_TRAP, "1", IRNode.RANGE_CHECK_TRAP, "3", IRNode.CLASS_CHECK_TRAP, "1", IRNode.IF, "5" }) + @IR(failOn = IRNode.ALLOC) + public static int test24(MyValue3[] array, int i, int j, int k) { + return array[i].intField1 + array[j].intField1 + array[k].intField1; + } + + @Run(test = "test24") + @Warmup(10_000) + public static void test24Runner() { + test24(array11, 0, 0, 0); + } + + @Test + @IR(counts = { IRNode.NULL_CHECK_TRAP, "4", IRNode.RANGE_CHECK_TRAP, "3", IRNode.IF, "7" }) + @IR(failOn = IRNode.ALLOC) + public static int test25(MyValue3[] array, int i, int j, int k) { + return array[i].intField1 + array[j].intField1 + array[k].intField1; + } + + @Run(test = "test25") + @Warmup(10_000) + public static void test25Runner() { + test25(array12, 0, 0, 0); + } + + @Test + @IR(counts = { IRNode.NULL_CHECK_TRAP, "1", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "3", IRNode.TRAP, "5", IRNode.CALL, "5", IRNode.IF, "6" }) + @IR(failOn = IRNode.ALLOC) + public static void test26() { + I[] array = array14; + test26Inline(array); + } + + @Run(test = "test26") + @Warmup(10_000) + public static void test26Runner() { + test26(); + test26Inline(array8); // flat, null free atomic + test26Inline(array18); // flat, null free atomic + } + + @ForceInline + static void test26Inline(I[] array) { + array[0].m(); + } + + @Test + @IR(counts = { IRNode.NULL_CHECK_TRAP, "1", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "3", IRNode.TRAP, "5", IRNode.CALL, "5", IRNode.IF, "6" }) + @IR(failOn = IRNode.ALLOC) + public static void test27() { + I[] array = array13; + test27Inline(array); + } + + @Run(test = "test27") + @Warmup(10_000) + public static void test27Runner() { + test27(); + test27Inline(array3); // flat, null free non atomic + test27Inline(array17); // flat, null free non atomic + } + + @ForceInline + static void test27Inline(I[] array) { + array[0].m(); + } + + @Test + @IR(counts = { IRNode.NULL_CHECK_TRAP, "3", IRNode.RANGE_CHECK_TRAP, "2", IRNode.CLASS_CHECK_TRAP, "1", IRNode.TRAP, "6", IRNode.CALL, "6", IRNode.IF, "6" }) + @IR(failOn = IRNode.ALLOC) + public static void test28(I[][] arrayOfArray) { + I[] array = arrayOfArray[0]; + test28Inline(array); + } + + @Run(test = "test28") + @Warmup(10_000) + public static void test28Runner() { + test28(array23); + test28Inline(array1); + test28Inline(array2); + test28Inline(array5); + } + + @ForceInline + static void test28Inline(I[] array) { + array[0].m(); + } + + @Test + @IR(counts = { IRNode.NULL_CHECK_TRAP, "3", IRNode.RANGE_CHECK_TRAP, "2", IRNode.CLASS_CHECK_TRAP, "1", IRNode.TRAP, "6", IRNode.CALL, "6", IRNode.IF, "6" }) + @IR(failOn = IRNode.ALLOC) + public static void test29(I[][] arrayOfArray) { + I[] array = arrayOfArray[0]; + test29Inline(array); + } + + @Run(test = "test29") + @Warmup(10_000) + public static void test29Runner() { + test29(array24); + test29Inline(array1); + test29Inline(array2); + test29Inline(array5); + } + + @ForceInline + static void test29Inline(I[] array) { + array[0].m(); + } + + @Test + @IR(counts = { IRNode.NULL_CHECK_TRAP, "3", IRNode.RANGE_CHECK_TRAP, "3", IRNode.CLASS_CHECK_TRAP, "1", IRNode.TRAP, "7", IRNode.CALL, "7", IRNode.IF, "9" }) + @IR(failOn = IRNode.ALLOC) + public static Object test30(boolean flag, I[][] arrayOfArray1, I[][] arrayOfArray2) { + I[] array = null; + if (flag) { + array = arrayOfArray1[0]; + } else { + array = arrayOfArray2[0]; + } + return test30Inline(array); + } + + @Run(test = "test30") + @Warmup(10_000) + public static void test30Runner() { + test30(true, array23, array25); + test30(false, array23, array25); + test30Inline(array1); + test30Inline(array2); + test30Inline(array5); + } + + @ForceInline + static Object test30Inline(I[] array) { + return array[0]; + } + + @Test + @IR(counts = { IRNode.NULL_CHECK_TRAP, "4", IRNode.RANGE_CHECK_TRAP, "3", IRNode.CLASS_CHECK_TRAP, "1", IRNode.TRAP, "8", IRNode.CALL, "8", IRNode.IF, "11" }) + @IR(failOn = IRNode.ALLOC) + public static void test31(boolean flag, I[][] arrayOfArray1, I[][] arrayOfArray2) { + I[] array = null; + if (flag) { + array = arrayOfArray1[0]; + } else { + array = arrayOfArray2[0]; + } + I i = test31Inline(array); + i.m(); + } + + @Run(test = "test31") + @Warmup(10_000) + public static void test31Runner() { + test31(true, array24, array26); + test31(false, array24, array26); + test31Inline(array1); + test31Inline(array2); + test31Inline(array5); + } + + @ForceInline + static I test31Inline(I[] array) { + return array[0]; + } + + @Test + @IR(counts = { IRNode.NULL_CHECK_TRAP, "3", IRNode.RANGE_CHECK_TRAP, "3", IRNode.CLASS_CHECK_TRAP, "2", IRNode.BIMORPHIC_TRAP, "1", IRNode.TRAP, "9", IRNode.CALL, "9", IRNode.IF, "12" }) + @IR(failOn = IRNode.ALLOC) + public static void test32(boolean flag, I[][] arrayOfArray1, I[][] arrayOfArray2) { + I[] array = null; + if (flag) { + array = arrayOfArray1[0]; + } else { + array = arrayOfArray2[0]; + } + I i = test32Inline(array); + i.m(); + } + + @Run(test = "test32") + @Warmup(10_000) + public static void test32Runner() { + test32(true, array27, array28); + test32(false, array27, array28); + test32Inline(array1); + test32Inline(array2); + test32Inline(array5); + } + + @ForceInline + static I test32Inline(I[] array) { + return array[0]; + } + + // @Test + // @IR(counts = { IRNode.NULL_CHECK_TRAP, "1", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "1", IRNode.TRAP, "3", IRNode.CALL, "3", IRNode.IF, "3" }) + // @IR(failOn = IRNode.ALLOC) + // public static void test28(I[] array) { + // array[0].m(); + // array[0].m(); + // } + + // @Run(test = "test28") + // public static void test28Runner() { + // test28(array5); + // } + + + // @Test + // // @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "1", IRNode.TRAP, "4", IRNode.CALL, "4", IRNode.IF, "4" }) + // // @IR(failOn = IRNode.ALLOC) + // public static I test26(I[] array) { + // return array[0]; + // } + + // @Run(test = "test26") + // public static void test26Runner() { + // // test26(array21); // not flat, nullable + // // test26(array20); // flat, nullable, atomic + // // test26(array19); // flat, nullable, atomic + // // test26(array14); // flat, null restricted atomic + // test26(array13); // flat, null restricted, non atomic + + // // test26(array14); + // // test26(array17); + // // test26(array18); + // // test26(array15); + // // test26(array16); + // } + + // @Test + // @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "2", IRNode.TRAP, "5", IRNode.CALL, "5", IRNode.IF, "9" }) + // @IR(failOn = IRNode.ALLOC) + // public static void test24(I[] array) { + // array[0].m(); + // } + + // @Run(test = "test24") + // public static void test24Runner() { + // test24(array3); + // test24(array10); + // } + + interface I { + void m(); + } + + + @LooselyConsistentValue + static value class MyValue1 implements I { + byte byteField; + byte byteField2; + int byteField3; + + MyValue1(byte byteField) { + this.byteField = byteField; + this.byteField2 = byteField; + this.byteField3 = byteField; + } + + public void m() { + } + } + + static value class MyValue2 implements I { + int intField; + + MyValue2(int intField) { + this.intField = intField; + } + + public void m() { + } + } + + static value class MyValue3 implements I { + int intField1; + int intField2; + int intField3; + int intField4; + + MyValue3(int intField) { + this.intField1 = intField; + this.intField2 = intField; + this.intField3 = intField; + this.intField4 = intField; + } + + public void m() { + } + } + + @LooselyConsistentValue + static value class MyValue4 implements I { + byte byteField; + byte byteField2; + + MyValue4(byte byteField) { + this.byteField = byteField; + this.byteField2 = byteField; + } + + public void m() { + } + } + + @LooselyConsistentValue + static value class MyValue5 implements I { + byte byteField; + byte byteField2; + + MyValue5(byte byteField) { + this.byteField = byteField; + this.byteField2 = byteField; + } + + public void m() { + } + } + + static class A implements I { + public void m() { + } + } + + static class B implements I { + public void m() { + } + } + + static value class MyValue6 implements I { + int intField1; + int intField2; + int intField3; + int intField4; + + MyValue6(int intField) { + this.intField1 = intField; + this.intField2 = intField; + this.intField3 = intField; + this.intField4 = intField; + } + + public void m() { + } + } +} diff --git a/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestLWorld.java b/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestLWorld.java index 87e8049fa6e..46625fee147 100644 --- a/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestLWorld.java +++ b/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestLWorld.java @@ -206,6 +206,8 @@ private static void call() {} private static final MyValue1 testValue1 = MyValue1.createWithFieldsInline(rI, rL); @NullRestricted private static final MyValue2 testValue2 = MyValue2.createWithFieldsInline(rI, rD); + @NullRestricted + private static final MyValue3 testValue3 = MyValue3.create(); protected long hash() { return testValue1.hash(); @@ -789,6 +791,13 @@ public void test20_verifier() { } } + private static final MyValue3[] testValue3Array = (MyValue3[])ValueClass.newNullRestrictedNonAtomicArray(MyValue3.class, 3, MyValue3.DEFAULT); + static { + for (int i = 0; i < 3; ++i) { + testValue3Array[i] = testValue3; + } + } + private static final NonValueClass[] testNonValueArray = new NonValueClass[42]; // Test load from (flattened) inline type array disguised as object array @@ -1315,6 +1324,9 @@ public Object[] test38(Object[] oa, Object o, int i1, int i2, int num) { case 6: result = testValue1Array2; break; + case 7: + result = testValue3Array; + break; } result[i1] = result[i2]; result[i2] = o; @@ -1354,6 +1366,8 @@ public void test38_verifier() { } result = test38(null, testValue1Array, index, index, 6); Asserts.assertEQ(testValue1, ((MyValue1[][])result)[index][index]); + result = test38(null, testValue3, index, index, 7); + Asserts.assertEQ(testValue3, result[index]); } @ForceInline diff --git a/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestLWorldProfiling.java b/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestLWorldProfiling.java index 651ad0a923c..0580b8fb4a9 100644 --- a/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestLWorldProfiling.java +++ b/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestLWorldProfiling.java @@ -113,6 +113,8 @@ public static void main(String[] args) { private static final MyValue1 testValue1 = MyValue1.createWithFieldsInline(rI, rL); @NullRestricted private static final MyValue2 testValue2 = MyValue2.createWithFieldsInline(rI, rD); + @NullRestricted + private static final MyValue3 testValue3 = MyValue3.create(); private static final MyValue1[] testValue1Array = (MyValue1[])ValueClass.newNullRestrictedNonAtomicArray(MyValue1.class, 1, MyValue1.DEFAULT); static { testValue1Array[0] = testValue1; @@ -121,6 +123,10 @@ public static void main(String[] args) { static { testValue2Array[0] = testValue2; } + private static final MyValue3[] testValue3Array = (MyValue3[])ValueClass.newNullRestrictedNonAtomicArray(MyValue3.class, 1, MyValue3.DEFAULT); + static { + testValue3Array[0] = testValue3; + } // Some non-value classes static class MyInteger extends Number { @@ -258,6 +264,8 @@ public void test3_verifier() { Asserts.assertEQ(testValue1, o); o = test3(testValue2Array); Asserts.assertEQ(testValue2, o); + o = test3(testValue3Array); + Asserts.assertEQ(testValue3, o); } @Test @@ -296,6 +304,10 @@ public void test5_verifier() { Asserts.assertEQ(testValue1, o); o = test5(testValue1NotFlatArray); Asserts.assertEQ(testValue1, o); + o = test5(testValue2Array); + Asserts.assertEQ(testValue2, o); + o = test5(testValue3Array); + Asserts.assertEQ(testValue3, o); } // Check that profile data that's useless at the aaload is @@ -378,7 +390,7 @@ public void test8_helper(Object arg) { @Test @IR(applyIf = {"UseArrayLoadStoreProfile", "true"}, - counts = {STATIC_CALL, "= 5", CLASS_CHECK_TRAP, "= 1", NULL_CHECK_TRAP, "= 2", + counts = {STATIC_CALL, "= 7", CLASS_CHECK_TRAP, "= 2", NULL_CHECK_TRAP, "= 2", RANGE_CHECK_TRAP, "= 1"}) @IR(applyIf = {"UseArrayLoadStoreProfile", "false"}, counts = {STATIC_CALL, "= 5", RANGE_CHECK_TRAP, "= 1", NULL_CHECK_TRAP, "= 2"})