diff --git a/.github/workflows/build-reusable.yml b/.github/workflows/build-reusable.yml index 1c71527296e04..c8affc011b3f9 100644 --- a/.github/workflows/build-reusable.yml +++ b/.github/workflows/build-reusable.yml @@ -32,6 +32,7 @@ jobs: name: Linux runs-on: ${{matrix.os}} strategy: + fail-fast: false matrix: include: - lto_flag: "" diff --git a/Source/JavaScriptCore/CMakeLists.txt b/Source/JavaScriptCore/CMakeLists.txt index 77ab7851ef7d9..d43eb98552552 100644 --- a/Source/JavaScriptCore/CMakeLists.txt +++ b/Source/JavaScriptCore/CMakeLists.txt @@ -1330,6 +1330,8 @@ set(JavaScriptCore_PRIVATE_FRAMEWORK_HEADERS runtime/InferredValueInlines.h runtime/InitializeThreading.h runtime/InlineAttribute.h + runtime/InlinePropertyKey.h + runtime/InlineStringCache.h runtime/Int16Array.h runtime/Int32Array.h runtime/Int8Array.h @@ -1525,6 +1527,7 @@ set(JavaScriptCore_PRIVATE_FRAMEWORK_HEADERS runtime/JSSourceCode.h runtime/JSSourceCodeInlines.h runtime/JSString.h + runtime/JSStringBuilder.h runtime/JSStringInlines.h runtime/JSStringIterator.h runtime/JSStringIteratorInlines.h diff --git a/Source/JavaScriptCore/builtins/BuiltinNames.cpp b/Source/JavaScriptCore/builtins/BuiltinNames.cpp index b94f7f24cb369..04a8091ea0734 100644 --- a/Source/JavaScriptCore/builtins/BuiltinNames.cpp +++ b/Source/JavaScriptCore/builtins/BuiltinNames.cpp @@ -61,11 +61,19 @@ SymbolImpl::StaticSymbolImpl polyProtoPrivateName { "PolyProto", SymbolImpl::s_f m_privateNameSet.add(symbol); \ } while (0); +#if USE(BUN_JSC_ADDITIONS) +#define INITIALIZE_WELL_KNOWN_SYMBOL_PUBLIC_TO_PRIVATE_ENTRY(name) \ + do { \ + SymbolImpl* symbol = static_cast(m_##name##Symbol.impl()); \ + m_wellKnownSymbolsMap.add(m_##name##SymbolPrivateIdentifier.string(), symbol); \ + } while (0); +#else #define INITIALIZE_WELL_KNOWN_SYMBOL_PUBLIC_TO_PRIVATE_ENTRY(name) \ do { \ SymbolImpl* symbol = static_cast(m_##name##Symbol.impl()); \ m_wellKnownSymbolsMap.add(m_##name##SymbolPrivateIdentifier.impl(), symbol); \ } while (0); +#endif WTF_MAKE_TZONE_ALLOCATED_IMPL(BuiltinNames); diff --git a/Source/JavaScriptCore/bytecode/GetByStatus.cpp b/Source/JavaScriptCore/bytecode/GetByStatus.cpp index 7797feae92cd0..af1e22ecbae58 100644 --- a/Source/JavaScriptCore/bytecode/GetByStatus.cpp +++ b/Source/JavaScriptCore/bytecode/GetByStatus.cpp @@ -482,8 +482,13 @@ GetByStatus GetByStatus::computeFor(JSGlobalObject* globalObject, const Structur if (set.isEmpty()) return GetByStatus(); +#if USE(BUN_JSC_ADDITIONS) + if (parseIndex(PropertyName(identifier.uid()))) + return GetByStatus(LikelyTakesSlowPath); +#else if (parseIndex(*identifier.uid())) return GetByStatus(LikelyTakesSlowPath); +#endif VM& vm = globalObject->vm(); auto attempToFold = [&]() -> std::optional { diff --git a/Source/JavaScriptCore/bytecode/InlineAccess.cpp b/Source/JavaScriptCore/bytecode/InlineAccess.cpp index b553c9922f00c..5974297ff08cc 100644 --- a/Source/JavaScriptCore/bytecode/InlineAccess.cpp +++ b/Source/JavaScriptCore/bytecode/InlineAccess.cpp @@ -365,6 +365,12 @@ bool InlineAccess::generateStringLength(PropertyInlineCache& propertyCache) auto done = jit.jump(); isRope.link(&jit); +#if USE(BUN_JSC_ADDITIONS) + // Inline small strings share the rope branch; their cell is only 16 bytes, + // so JSRopeString::offsetOfLength() would read past the end. Take the slow path. + jit.branchTestPtr(CCallHelpers::Zero, scratch, CCallHelpers::TrustedImm32(JSString::isRopeInPointer)) + .linkThunk(propertyCache.slowPathStartLocation, &jit); +#endif jit.load32(CCallHelpers::Address(base, JSRopeString::offsetOfLength()), value.payloadGPR()); done.link(&jit); diff --git a/Source/JavaScriptCore/bytecode/InlineCacheCompiler.cpp b/Source/JavaScriptCore/bytecode/InlineCacheCompiler.cpp index f4b9d517a6c53..ff0e78a731c02 100644 --- a/Source/JavaScriptCore/bytecode/InlineCacheCompiler.cpp +++ b/Source/JavaScriptCore/bytecode/InlineCacheCompiler.cpp @@ -1875,11 +1875,22 @@ void InlineCacheCompiler::generateWithGuard(unsigned index, AccessCase& accessCa GPRReg propertyGPR = m_propertyCache.propertyGPR(); // non-rope string check done inside polymorphic access. +#if USE(BUN_JSC_ADDITIONS) + UniquedStringImpl* icUid = accessCase.uid(); + if (uidIsSymbol(icUid)) + jit.loadPtr(MacroAssembler::Address(propertyGPR, Symbol::offsetOfSymbolImpl()), scratchGPR); + else + jit.loadPtr(MacroAssembler::Address(propertyGPR, JSString::offsetOfValue()), scratchGPR); + // If the cached key is a canonical fiber word, an incoming atom-backed + // JSString for the same content won't pointer-match — slow path handles it. + fallThrough.append(jit.branchPtr(CCallHelpers::NotEqual, scratchGPR, CCallHelpers::TrustedImmPtr(icUid))); +#else if (accessCase.uid()->isSymbol()) jit.loadPtr(MacroAssembler::Address(propertyGPR, Symbol::offsetOfSymbolImpl()), scratchGPR); else jit.loadPtr(MacroAssembler::Address(propertyGPR, JSString::offsetOfValue()), scratchGPR); fallThrough.append(jit.branchPtr(CCallHelpers::NotEqual, scratchGPR, CCallHelpers::TrustedImmPtr(accessCase.uid()))); +#endif } auto emitDefaultGuard = [&] () { @@ -3961,8 +3972,19 @@ void InlineCacheCompiler::generateAccessCase(unsigned index, AccessCase& accessC auto done = jit.jump(); isRope.link(&jit); +#if USE(BUN_JSC_ADDITIONS) + // Inline small strings: length is in bits 3..7 of m_fiber (still in scratchGPR). + auto realRope = jit.branchTestPtr(CCallHelpers::NonZero, scratchGPR, CCallHelpers::TrustedImm32(JSString::isRopeInPointer)); + jit.urshiftPtr(CCallHelpers::TrustedImm32(JSString::inlineLengthShift), scratchGPR); + jit.and32(CCallHelpers::TrustedImm32(JSString::inlineLengthMask), scratchGPR, valueRegs.payloadGPR()); + auto doneInline = jit.jump(); + realRope.link(&jit); +#endif jit.load32(CCallHelpers::Address(baseGPR, JSRopeString::offsetOfLength()), valueRegs.payloadGPR()); +#if USE(BUN_JSC_ADDITIONS) + doneInline.link(&jit); +#endif done.link(&jit); jit.boxInt32(valueRegs.payloadGPR(), valueRegs); succeed(); @@ -5118,7 +5140,7 @@ AccessGenerationResult InlineCacheCompiler::compile(const GCSafeConcurrentJSLock if (!hasConstantIdentifier) { if (entry->requiresIdentifierNameMatch()) { - if (entry->uid()->isSymbol()) + if (uidIsSymbol(entry->uid())) needsSymbolPropertyCheck = true; else needsStringPropertyCheck = true; @@ -5203,7 +5225,7 @@ AccessGenerationResult InlineCacheCompiler::compile(const GCSafeConcurrentJSLock for (unsigned i = keys.size(); i--;) { fallThrough.link(&jit); fallThrough.shrink(0); - if (keys[i]->requiresIdentifierNameMatch() && !keys[i]->uid()->isSymbol()) + if (keys[i]->requiresIdentifierNameMatch() && !uidIsSymbol(keys[i]->uid())) generateWithGuard(i, keys[i].get(), fallThrough); } @@ -5232,7 +5254,7 @@ AccessGenerationResult InlineCacheCompiler::compile(const GCSafeConcurrentJSLock for (unsigned i = keys.size(); i--;) { fallThrough.link(&jit); fallThrough.shrink(0); - if (keys[i]->requiresIdentifierNameMatch() && keys[i]->uid()->isSymbol()) + if (keys[i]->requiresIdentifierNameMatch() && uidIsSymbol(keys[i]->uid())) generateWithGuard(i, keys[i].get(), fallThrough); } @@ -7704,7 +7726,7 @@ AccessGenerationResult InlineCacheCompiler::compileOneAccessCaseHandler(const Ve switch (accessCase.m_type) { case AccessCase::GetGetter: case AccessCase::Load: - code = vm.getCTIStub(accessCase.uid()->isSymbol() ? CommonJITThunkID::GetByValWithSymbolLoadOwnPropertyHandler : CommonJITThunkID::GetByValWithStringLoadOwnPropertyHandler).retagged(); + code = vm.getCTIStub(uidIsSymbol(accessCase.uid()) ? CommonJITThunkID::GetByValWithSymbolLoadOwnPropertyHandler : CommonJITThunkID::GetByValWithStringLoadOwnPropertyHandler).retagged(); break; case AccessCase::IndexedUndefinedKeyLoad: code = vm.getCTIStub(CommonJITThunkID::GetByValWithUndefinedKeyLoadOwnPropertyHandler).retagged(); @@ -7725,7 +7747,7 @@ AccessGenerationResult InlineCacheCompiler::compileOneAccessCaseHandler(const Ve switch (accessCase.m_type) { case AccessCase::GetGetter: case AccessCase::Load: - code = vm.getCTIStub(accessCase.uid()->isSymbol() ? CommonJITThunkID::GetByValWithSymbolLoadPrototypePropertyHandler : CommonJITThunkID::GetByValWithStringLoadPrototypePropertyHandler).retagged(); + code = vm.getCTIStub(uidIsSymbol(accessCase.uid()) ? CommonJITThunkID::GetByValWithSymbolLoadPrototypePropertyHandler : CommonJITThunkID::GetByValWithStringLoadPrototypePropertyHandler).retagged(); break; case AccessCase::IndexedUndefinedKeyLoad: code = vm.getCTIStub(CommonJITThunkID::GetByValWithUndefinedKeyLoadPrototypePropertyHandler).retagged(); @@ -7762,7 +7784,7 @@ AccessGenerationResult InlineCacheCompiler::compileOneAccessCaseHandler(const Ve MacroAssemblerCodeRef code; switch (accessCase.m_type) { case AccessCase::Miss: - code = vm.getCTIStub(accessCase.uid()->isSymbol() ? CommonJITThunkID::GetByValWithSymbolMissHandler : CommonJITThunkID::GetByValWithStringMissHandler).retagged(); + code = vm.getCTIStub(uidIsSymbol(accessCase.uid()) ? CommonJITThunkID::GetByValWithSymbolMissHandler : CommonJITThunkID::GetByValWithStringMissHandler).retagged(); break; case AccessCase::IndexedUndefinedKeyMiss: code = vm.getCTIStub(CommonJITThunkID::GetByValWithUndefinedKeyMissHandler).retagged(); @@ -7810,17 +7832,17 @@ AccessGenerationResult InlineCacheCompiler::compileOneAccessCaseHandler(const Ve MacroAssemblerCodeRef code; if (accessCase.m_type == AccessCase::CustomAccessorGetter) { if (Options::useDOMJIT() && access.domAttribute() && access.domAttribute()->domJIT) { - code = compileGetByDOMJITHandler(codeBlock, access.domAttribute()->domJIT, accessCase.uid()->isSymbol()); + code = compileGetByDOMJITHandler(codeBlock, access.domAttribute()->domJIT, uidIsSymbol(accessCase.uid())); if (!code) return AccessGenerationResult::GaveUp; } else { - if (accessCase.uid()->isSymbol()) + if (uidIsSymbol(accessCase.uid())) code = vm.getCTIStub(CommonJITThunkID::GetByValWithSymbolCustomAccessorHandler).retagged(); else code = vm.getCTIStub(CommonJITThunkID::GetByValWithStringCustomAccessorHandler).retagged(); } } else { - if (accessCase.uid()->isSymbol()) + if (uidIsSymbol(accessCase.uid())) code = vm.getCTIStub(CommonJITThunkID::GetByValWithSymbolCustomValueHandler).retagged(); else code = vm.getCTIStub(CommonJITThunkID::GetByValWithStringCustomValueHandler).retagged(); @@ -7842,7 +7864,7 @@ AccessGenerationResult InlineCacheCompiler::compileOneAccessCaseHandler(const Ve currStructure = object->structure(); if (isValidOffset(accessCase.m_offset)) currStructure->startWatchingPropertyForReplacements(vm, accessCase.offset()); - auto code = vm.getCTIStub(accessCase.uid()->isSymbol() ? CommonJITThunkID::GetByValWithSymbolGetterHandler : CommonJITThunkID::GetByValWithStringGetterHandler).retagged(); + auto code = vm.getCTIStub(uidIsSymbol(accessCase.uid()) ? CommonJITThunkID::GetByValWithSymbolGetterHandler : CommonJITThunkID::GetByValWithStringGetterHandler).retagged(); auto stub = createPreCompiledICJITStubRoutine(WTF::move(code), vm, codeBlock); connectWatchpointSets(stub.get(), WTF::move(watchedConditions), WTF::move(additionalWatchpointSets)); return finishPreCompiledCodeGeneration(WTF::move(stub)); @@ -7875,7 +7897,7 @@ AccessGenerationResult InlineCacheCompiler::compileOneAccessCaseHandler(const Ve MacroAssemblerCodeRef code; switch (accessCase.m_type) { case AccessCase::Replace: - code = vm.getCTIStub(accessCase.uid()->isSymbol() ? CommonJITThunkID::PutByValWithSymbolReplaceHandler : CommonJITThunkID::PutByValWithStringReplaceHandler).retagged(); + code = vm.getCTIStub(uidIsSymbol(accessCase.uid()) ? CommonJITThunkID::PutByValWithSymbolReplaceHandler : CommonJITThunkID::PutByValWithStringReplaceHandler).retagged(); break; case AccessCase::IndexedUndefinedKeyReplace: code = vm.getCTIStub(CommonJITThunkID::PutByValWithUndefinedKeyReplaceHandler).retagged(); @@ -7922,7 +7944,7 @@ AccessGenerationResult InlineCacheCompiler::compileOneAccessCaseHandler(const Ve MacroAssemblerCodeRef code; switch (accessCase.m_type) { case AccessCase::Transition: - if (accessCase.uid()->isSymbol()) + if (uidIsSymbol(accessCase.uid())) code = selectTransitionHandler(CommonJITThunkID::PutByValWithSymbolTransitionNonAllocatingHandler, CommonJITThunkID::PutByValWithSymbolTransitionReallocatingOutOfLineHandler, CommonJITThunkID::PutByValWithSymbolTransitionNewlyAllocatingHandler, CommonJITThunkID::PutByValWithSymbolTransitionReallocatingHandler); else code = selectTransitionHandler(CommonJITThunkID::PutByValWithStringTransitionNonAllocatingHandler, CommonJITThunkID::PutByValWithStringTransitionReallocatingOutOfLineHandler, CommonJITThunkID::PutByValWithStringTransitionNewlyAllocatingHandler, CommonJITThunkID::PutByValWithStringTransitionReallocatingHandler); @@ -7962,12 +7984,12 @@ AccessGenerationResult InlineCacheCompiler::compileOneAccessCaseHandler(const Ve MacroAssemblerCodeRef code; if (accessCase.m_type == AccessCase::CustomAccessorSetter) { - if (accessCase.uid()->isSymbol()) + if (uidIsSymbol(accessCase.uid())) code = vm.getCTIStub(CommonJITThunkID::PutByValWithSymbolCustomAccessorHandler).retagged(); else code = vm.getCTIStub(CommonJITThunkID::PutByValWithStringCustomAccessorHandler).retagged(); } else { - if (accessCase.uid()->isSymbol()) + if (uidIsSymbol(accessCase.uid())) code = vm.getCTIStub(CommonJITThunkID::PutByValWithSymbolCustomValueHandler).retagged(); else code = vm.getCTIStub(CommonJITThunkID::PutByValWithStringCustomValueHandler).retagged(); @@ -7992,9 +8014,9 @@ AccessGenerationResult InlineCacheCompiler::compileOneAccessCaseHandler(const Ve MacroAssemblerCodeRef code; if (isStrict) - code = vm.getCTIStub(accessCase.uid()->isSymbol() ? CommonJITThunkID::PutByValWithSymbolStrictSetterHandler : CommonJITThunkID::PutByValWithStringStrictSetterHandler).retagged(); + code = vm.getCTIStub(uidIsSymbol(accessCase.uid()) ? CommonJITThunkID::PutByValWithSymbolStrictSetterHandler : CommonJITThunkID::PutByValWithStringStrictSetterHandler).retagged(); else - code = vm.getCTIStub(accessCase.uid()->isSymbol() ? CommonJITThunkID::PutByValWithSymbolSloppySetterHandler : CommonJITThunkID::PutByValWithStringSloppySetterHandler).retagged(); + code = vm.getCTIStub(uidIsSymbol(accessCase.uid()) ? CommonJITThunkID::PutByValWithSymbolSloppySetterHandler : CommonJITThunkID::PutByValWithStringSloppySetterHandler).retagged(); auto stub = createPreCompiledICJITStubRoutine(WTF::move(code), vm, codeBlock); connectWatchpointSets(stub.get(), WTF::move(watchedConditions), WTF::move(additionalWatchpointSets)); return finishPreCompiledCodeGeneration(WTF::move(stub)); @@ -8018,7 +8040,7 @@ AccessGenerationResult InlineCacheCompiler::compileOneAccessCaseHandler(const Ve collectConditions(accessCase, watchedConditions, checkingConditions); if (checkingConditions.isEmpty()) { MacroAssemblerCodeRef code; - if (accessCase.uid()->isSymbol()) + if (uidIsSymbol(accessCase.uid())) code = vm.getCTIStub(accessCase.m_type == AccessCase::InHit ? CommonJITThunkID::InByValWithSymbolHitHandler : CommonJITThunkID::InByValWithSymbolMissHandler).retagged(); else code = vm.getCTIStub(accessCase.m_type == AccessCase::InHit ? CommonJITThunkID::InByValWithStringHitHandler : CommonJITThunkID::InByValWithStringMissHandler).retagged(); @@ -8045,13 +8067,13 @@ AccessGenerationResult InlineCacheCompiler::compileOneAccessCaseHandler(const Ve CommonJITThunkID thunkID = CommonJITThunkID::DeleteByValWithStringDeleteHandler; switch (accessCase.m_type) { case AccessCase::Delete: - thunkID = accessCase.uid()->isSymbol() ? CommonJITThunkID::DeleteByValWithSymbolDeleteHandler : CommonJITThunkID::DeleteByValWithStringDeleteHandler; + thunkID = uidIsSymbol(accessCase.uid()) ? CommonJITThunkID::DeleteByValWithSymbolDeleteHandler : CommonJITThunkID::DeleteByValWithStringDeleteHandler; break; case AccessCase::DeleteNonConfigurable: - thunkID = accessCase.uid()->isSymbol() ? CommonJITThunkID::DeleteByValWithSymbolDeleteNonConfigurableHandler : CommonJITThunkID::DeleteByValWithStringDeleteNonConfigurableHandler; + thunkID = uidIsSymbol(accessCase.uid()) ? CommonJITThunkID::DeleteByValWithSymbolDeleteNonConfigurableHandler : CommonJITThunkID::DeleteByValWithStringDeleteNonConfigurableHandler; break; case AccessCase::DeleteMiss: - thunkID = accessCase.uid()->isSymbol() ? CommonJITThunkID::DeleteByValWithSymbolDeleteMissHandler : CommonJITThunkID::DeleteByValWithStringDeleteMissHandler; + thunkID = uidIsSymbol(accessCase.uid()) ? CommonJITThunkID::DeleteByValWithSymbolDeleteMissHandler : CommonJITThunkID::DeleteByValWithStringDeleteMissHandler; break; default: break; @@ -8148,7 +8170,7 @@ AccessGenerationResult InlineCacheCompiler::compileOneAccessCaseHandler(const Ve #endif } m_failAndRepatch.append(notInt32); - } else if (accessCase.requiresIdentifierNameMatch() && !accessCase.uid()->isSymbol()) { + } else if (accessCase.requiresIdentifierNameMatch() && !uidIsSymbol(accessCase.uid())) { CCallHelpers::JumpList notString; GPRReg propertyGPR = m_propertyCache.propertyGPR(); if (!m_propertyCache.propertyIsString) { @@ -8163,7 +8185,7 @@ AccessGenerationResult InlineCacheCompiler::compileOneAccessCaseHandler(const Ve jit.loadPtr(MacroAssembler::Address(propertyGPR, JSString::offsetOfValue()), m_scratchGPR); m_failAndRepatch.append(jit.branchIfRopeStringImpl(m_scratchGPR)); m_failAndRepatch.append(notString); - } else if (accessCase.requiresIdentifierNameMatch() && accessCase.uid()->isSymbol()) { + } else if (accessCase.requiresIdentifierNameMatch() && uidIsSymbol(accessCase.uid())) { CCallHelpers::JumpList notSymbol; if (!m_propertyCache.propertyIsSymbol) { GPRReg propertyGPR = m_propertyCache.propertyGPR(); diff --git a/Source/JavaScriptCore/bytecode/InlineCacheCompiler.h b/Source/JavaScriptCore/bytecode/InlineCacheCompiler.h index 8acdc62771d2c..fc7ee1b45a3ee 100644 --- a/Source/JavaScriptCore/bytecode/InlineCacheCompiler.h +++ b/Source/JavaScriptCore/bytecode/InlineCacheCompiler.h @@ -157,7 +157,11 @@ ALWAYS_INLINE bool canUseMegamorphicGetByIdExcludingIndex(VM& vm, UniquedStringI inline bool canUseMegamorphicGetById(VM& vm, UniquedStringImpl* uid) { +#if USE(BUN_JSC_ADDITIONS) + return !parseIndex(PropertyName(uid)) && canUseMegamorphicGetByIdExcludingIndex(vm, uid); +#else return !parseIndex(*uid) && canUseMegamorphicGetByIdExcludingIndex(vm, uid); +#endif } inline bool canUseMegamorphicInById(VM& vm, UniquedStringImpl* uid) @@ -167,7 +171,11 @@ inline bool canUseMegamorphicInById(VM& vm, UniquedStringImpl* uid) inline bool canUseMegamorphicPutById(VM& vm, UniquedStringImpl* uid) { +#if USE(BUN_JSC_ADDITIONS) + return !parseIndex(PropertyName(uid)) && uid != vm.propertyNames->underscoreProto; +#else return !parseIndex(*uid) && uid != vm.propertyNames->underscoreProto; +#endif } bool NODELETE canBeViaGlobalProxy(AccessCase::AccessType); diff --git a/Source/JavaScriptCore/bytecode/PutByStatus.cpp b/Source/JavaScriptCore/bytecode/PutByStatus.cpp index 94e0972ae8414..dbe001a72c584 100644 --- a/Source/JavaScriptCore/bytecode/PutByStatus.cpp +++ b/Source/JavaScriptCore/bytecode/PutByStatus.cpp @@ -369,8 +369,13 @@ PutByStatus PutByStatus::computeFor(CodeBlock* baselineBlock, ICStatusMap& basel PutByStatus PutByStatus::computeFor(JSGlobalObject* globalObject, const StructureSet& set, CacheableIdentifier identifier, bool isDirect, PrivateFieldPutKind privateFieldPutKind) { UniquedStringImpl* uid = identifier.uid(); +#if USE(BUN_JSC_ADDITIONS) + if (parseIndex(PropertyName(uid))) + return PutByStatus(LikelyTakesSlowPath); +#else if (parseIndex(*uid)) return PutByStatus(LikelyTakesSlowPath); +#endif if (set.isEmpty()) return PutByStatus(); diff --git a/Source/JavaScriptCore/bytecode/SpeculatedType.cpp b/Source/JavaScriptCore/bytecode/SpeculatedType.cpp index 8532f1c6df047..d820589d0f238 100644 --- a/Source/JavaScriptCore/bytecode/SpeculatedType.cpp +++ b/Source/JavaScriptCore/bytecode/SpeculatedType.cpp @@ -271,6 +271,13 @@ void dumpSpeculation(PrintStream& outStream, SpeculatedType value) strOut.print("StringUnresolvedVar"); else isTop = false; + +#if USE(BUN_JSC_ADDITIONS) + if (value & SpecStringInline) + strOut.print("StringInline"); + else + isTop = false; +#endif } } @@ -610,6 +617,10 @@ SpeculatedType speculationFromCell(JSCell* cell) return SpecStringIdent; return SpecStringResolved; } +#if USE(BUN_JSC_ADDITIONS) + if (string->isInline()) + return SpecStringInline; +#endif return SpecString; } diff --git a/Source/JavaScriptCore/bytecode/SpeculatedType.h b/Source/JavaScriptCore/bytecode/SpeculatedType.h index eca1fd21fab2b..82943b16519be 100644 --- a/Source/JavaScriptCore/bytecode/SpeculatedType.h +++ b/Source/JavaScriptCore/bytecode/SpeculatedType.h @@ -80,7 +80,12 @@ static constexpr SpeculatedType SpecObjectOther = 1ull << static constexpr SpeculatedType SpecStringIdent = 1ull << 32; // It's definitely a JSString, and it's an identifier. static constexpr SpeculatedType SpecStringResolvedVar = 1ull << 33; // It's definitely a JSString, and it's not an identifier. And string is resolved. static constexpr SpeculatedType SpecStringUnresolvedVar = 1ull << 34; // It's definitely a JSString, and it's not an identifier. And string is unresolved. +#if USE(BUN_JSC_ADDITIONS) +static constexpr SpeculatedType SpecStringInline = 1ull << 51; // It's definitely a JSString with inline-small-string storage (not resolved, not a rope). +static constexpr SpeculatedType SpecStringVar = SpecStringUnresolvedVar | SpecStringResolvedVar | SpecStringInline; // It's definitely a JSString, and it's not an identifier. +#else static constexpr SpeculatedType SpecStringVar = SpecStringUnresolvedVar | SpecStringResolvedVar; // It's definitely a JSString, and it's not an identifier. +#endif static constexpr SpeculatedType SpecStringResolved = SpecStringIdent | SpecStringResolvedVar; // It's definitely a JSString, and it's resolved. May be an identifier or not. static constexpr SpeculatedType SpecString = SpecStringIdent | SpecStringVar; // It's definitely a JSString. static constexpr SpeculatedType SpecSymbol = 1ull << 35; // It's definitely a Symbol. diff --git a/Source/JavaScriptCore/bytecode/UnlinkedFunctionExecutable.h b/Source/JavaScriptCore/bytecode/UnlinkedFunctionExecutable.h index b59b2a846eaa4..6da14f3940768 100644 --- a/Source/JavaScriptCore/bytecode/UnlinkedFunctionExecutable.h +++ b/Source/JavaScriptCore/bytecode/UnlinkedFunctionExecutable.h @@ -353,7 +353,12 @@ class UnlinkedFunctionExecutable final : public JSCell { }; #if !ASSERT_ENABLED +#if USE(BUN_JSC_ADDITIONS) +// Identifier carries a lazily-materialized AtomString under BUN_JSC_ADDITIONS (phase D.1), which grows this type. +static_assert(sizeof(UnlinkedFunctionExecutable) <= 112, "UnlinkedFunctionExecutable needs to be small"); +#else static_assert(sizeof(UnlinkedFunctionExecutable) <= 96, "UnlinkedFunctionExecutable needs to be small"); #endif +#endif } // namespace JSC diff --git a/Source/JavaScriptCore/bytecompiler/BytecodeGenerator.cpp b/Source/JavaScriptCore/bytecompiler/BytecodeGenerator.cpp index abee238e608c9..fa9a1ffafd512 100644 --- a/Source/JavaScriptCore/bytecompiler/BytecodeGenerator.cpp +++ b/Source/JavaScriptCore/bytecompiler/BytecodeGenerator.cpp @@ -4528,6 +4528,24 @@ void BytecodeGenerator::endSwitch(const Vector, 8>& labels, Expressio ASSERT(!labels[i]->isForward()); ASSERT(nodes[i]->isString()); +#if USE(BUN_JSC_ADDITIONS) + // StringOffsetTable is RefPtr-keyed and the runtime lookup + // compares real StringImpl*, so materialize the fiber word here instead + // of letting a tagged pointer reach DefaultRefDerefTraits::refIfNotNull. + const Identifier& caseIdent = static_cast(nodes[i])->value(); + // string() is by-value now; keep the atom alive across add() so a + // fiber-word materialization survives until the table takes its ref. + AtomString clauseAtom = caseIdent.string(); + UniquedStringImpl* clause = clauseAtom.impl(); + ASSERT(!isInlinePropertyKey(clause)); + ASSERT(clause->isAtom()); + auto result = jumpTable.m_offsetTable.add(clause, UnlinkedStringJumpTable::OffsetLocation { labels[i]->bind(switchInfo.bytecodeOffset), 0 }); + if (result.isNewEntry) { + result.iterator->value.m_indexInTable = jumpTable.m_offsetTable.size() - 1; + jumpTable.m_minLength = std::min(jumpTable.m_minLength, clause->length()); + jumpTable.m_maxLength = std::max(jumpTable.m_maxLength, clause->length()); + } +#else UniquedStringImpl* clause = static_cast(nodes[i])->value().impl(); ASSERT(clause->isAtom()); auto result = jumpTable.m_offsetTable.add(clause, UnlinkedStringJumpTable::OffsetLocation { labels[i]->bind(switchInfo.bytecodeOffset), 0 }); @@ -4536,6 +4554,7 @@ void BytecodeGenerator::endSwitch(const Vector, 8>& labels, Expressio jumpTable.m_minLength = std::min(jumpTable.m_minLength, clause->length()); jumpTable.m_maxLength = std::max(jumpTable.m_maxLength, clause->length()); } +#endif } ASSERT(!defaultLabel.isForward()); jumpTable.m_defaultOffset = defaultLabel.bind(switchInfo.bytecodeOffset); @@ -5046,11 +5065,19 @@ RegisterID* BytecodeGenerator::emitGetTemplateObject(RegisterID* dst, TaggedTemp for (; templateString; templateString = templateString->next()) { auto* string = templateString->value(); ASSERT(string->raw()); +#if USE(BUN_JSC_ADDITIONS) + rawStrings.append(string->raw()->string()); + if (!string->cooked()) + cookedStrings.append(std::nullopt); + else + cookedStrings.append(string->cooked()->string()); +#else rawStrings.append(string->raw()->impl()); if (!string->cooked()) cookedStrings.append(std::nullopt); else cookedStrings.append(string->cooked()->impl()); +#endif } RefPtr constant = addTemplateObjectConstant(TemplateObjectDescriptor::create(WTF::move(rawStrings), WTF::move(cookedStrings)), taggedTemplate->endOffset()); if (!dst) diff --git a/Source/JavaScriptCore/bytecompiler/BytecodeGenerator.h b/Source/JavaScriptCore/bytecompiler/BytecodeGenerator.h index f1a6e7e0b670c..adf1f138df626 100644 --- a/Source/JavaScriptCore/bytecompiler/BytecodeGenerator.h +++ b/Source/JavaScriptCore/bytecompiler/BytecodeGenerator.h @@ -1280,7 +1280,11 @@ namespace JSC { Optimize, DoNotOptimize }; +#if USE(BUN_JSC_ADDITIONS) + typedef UncheckedKeyHashMap TDZMap; +#else typedef UncheckedKeyHashMap, TDZNecessityLevel, IdentifierRepHash> TDZMap; +#endif public: JSString* addStringConstant(const Identifier&); @@ -1355,7 +1359,11 @@ namespace JSC { // Some of these objects keep pointers to one another. They are arranged // to ensure a sane destruction order that avoids references to freed memory. +#if USE(BUN_JSC_ADDITIONS) + UncheckedKeyHashSet m_functions; +#else UncheckedKeyHashSet, IdentifierRepHash> m_functions; +#endif RegisterID m_ignoredResultRegister; RegisterID m_thisRegister; RegisterID m_calleeRegister; diff --git a/Source/JavaScriptCore/bytecompiler/NodesCodegen.cpp b/Source/JavaScriptCore/bytecompiler/NodesCodegen.cpp index 1e187661af8a9..de6e6ffab409a 100644 --- a/Source/JavaScriptCore/bytecompiler/NodesCodegen.cpp +++ b/Source/JavaScriptCore/bytecompiler/NodesCodegen.cpp @@ -4929,9 +4929,9 @@ static void processClauseList(ClauseListNode* list, Vector& typeForTable = SwitchNeither; break; } - auto& value = static_cast(clauseExpression)->value().string(); + const Identifier& value = static_cast(clauseExpression)->value(); if (singleCharacterSwitch &= value.length() == 1) { - int32_t intVal = value[0]; + int32_t intVal = value.string()[0]; if (intVal < min_num) min_num = intVal; if (intVal > max_num) diff --git a/Source/JavaScriptCore/dfg/DFGAbstractInterpreterInlines.h b/Source/JavaScriptCore/dfg/DFGAbstractInterpreterInlines.h index 2acbc4bfd89cd..de29f47194d85 100644 --- a/Source/JavaScriptCore/dfg/DFGAbstractInterpreterInlines.h +++ b/Source/JavaScriptCore/dfg/DFGAbstractInterpreterInlines.h @@ -5025,7 +5025,14 @@ bool AbstractInterpreter::executeEffects(unsigned clobberLimi case ResolveRope: { AbstractValue& value = forNode(node->child1()); JSValue childConst = value.value(); +#if USE(BUN_JSC_ADDITIONS) + // Inline small strings are !isRope() but still lack a StringImpl*. Folding them through + // lets the generic constant folder elide this node, leaking an inline fiber to consumers + // (GetByVal/StringCharAt/StringCodePointAt) that load StringImpl fields without a guard. + if (childConst && childConst.isString() && !asString(childConst)->isRope() && !asString(childConst)->isInline()) { +#else if (childConst && childConst.isString() && !asString(childConst)->isRope()) { +#endif setConstant(node, *m_graph.freeze(childConst)); break; } diff --git a/Source/JavaScriptCore/dfg/DFGFixupPhase.cpp b/Source/JavaScriptCore/dfg/DFGFixupPhase.cpp index 3871f97cef1c9..33c3cddd182dc 100644 --- a/Source/JavaScriptCore/dfg/DFGFixupPhase.cpp +++ b/Source/JavaScriptCore/dfg/DFGFixupPhase.cpp @@ -1140,6 +1140,19 @@ class FixupPhase : public Phase { ASSERT(node->arrayMode() == ArrayMode(Array::String, Array::Read, Array::OutOfBounds) || node->arrayMode() == ArrayMode(Array::String, Array::Read, Array::InBounds)); else ASSERT(node->arrayMode() == ArrayMode(Array::String, Array::Read)); +#if USE(BUN_JSC_ADDITIONS) + // For StringCharCodeAt/StringCodePointAt on a monomorphic-inline + // profile, skip the ResolveRope insertion so the compile functions + // read from the cell bytes directly without a StringImpl. + { + SpeculatedType pred = node->child1()->prediction() & SpecString; + if ((op == StringCharCodeAt || op == StringCodePointAt) && pred && !(pred & ~SpecStringInline)) { + fixEdge(node->child1()); + fixEdge(node->child2()); + break; + } + } +#endif blessArrayOperation(node->child1(), node->child2(), node->child1()); // Rewrite child1 with ResolveRope. fixEdge(node->child1()); fixEdge(node->child2()); @@ -2615,7 +2628,7 @@ class FixupPhase : public Phase { } case CheckIdent: { - if (node->uidOperand()->isSymbol()) + if (uidIsSymbol(node->uidOperand())) fixEdge(node->child1()); else fixEdge(node->child1()); diff --git a/Source/JavaScriptCore/dfg/DFGOperations.cpp b/Source/JavaScriptCore/dfg/DFGOperations.cpp index 8c19646ba8fb7..ccaf04c1b8e23 100644 --- a/Source/JavaScriptCore/dfg/DFGOperations.cpp +++ b/Source/JavaScriptCore/dfg/DFGOperations.cpp @@ -49,6 +49,9 @@ #include "FTLOSREntry.h" #include "FrameTracers.h" #include "HasOwnPropertyCache.h" +#if USE(BUN_JSC_ADDITIONS) +#include "InlinePropertyKey.h" +#endif #include "Interpreter.h" #include "InterpreterInlines.h" #include "IntlCollator.h" @@ -919,10 +922,27 @@ JSC_DEFINE_JIT_OPERATION(operationGetByValObjectString, EncodedJSValue, (JSGloba auto scope = DECLARE_THROW_SCOPE(vm); +#if USE(BUN_JSC_ADDITIONS) + // D.4 coherence + perf: grab the canonical fiber word straight from an + // inline cell's m_fiber (no toAtomString round-trip, m_fiber left unmutated). + GCOwnedDataScope propertyName; + UniquedStringImpl* uid; + if (uintptr_t fiber = asString(string)->tryGetCanonicalInlineFiberWord()) + uid = reinterpret_cast(fiber); + else { + propertyName = asString(string)->toAtomString(globalObject); + OPERATION_RETURN_IF_EXCEPTION(scope, encodedJSValue()); + uid = propertyName.data; + if (uintptr_t fiber = Identifier::canonicalFiberWordFor(uid)) + uid = reinterpret_cast(fiber); + } + OPERATION_RETURN(scope, JSValue::encode(getByValObject(globalObject, vm, asObject(base), uid))); +#else auto propertyName = asString(string)->toAtomString(globalObject); OPERATION_RETURN_IF_EXCEPTION(scope, encodedJSValue()); OPERATION_RETURN(scope, JSValue::encode(getByValObject(globalObject, vm, asObject(base), propertyName.data))); +#endif } JSC_DEFINE_JIT_OPERATION(operationGetByValObjectSymbol, EncodedJSValue, (JSGlobalObject* globalObject, JSCell* base, JSCell* symbol)) @@ -4800,32 +4820,82 @@ JSC_DEFINE_JIT_OPERATION(operationSwitchString, char*, (JSGlobalObject* globalOb JITOperationPrologueCallFrameTracer tracer(vm, callFrame); auto scope = DECLARE_THROW_SCOPE(vm); + CodeBlock* codeBlock = callFrame->codeBlock(); + const StringJumpTable& linkedTable = codeBlock->dfgStringSwitchJumpTable(tableIndex); +#if USE(BUN_JSC_ADDITIONS) + if (string->isInline()) { + unsigned length = string->length(); + RefPtr atom = string->is8Bit() + ? AtomStringImpl::lookUp(std::span { string->inlineData8(), length }) + : AtomStringImpl::lookUp(std::span { string->inlineData16(), length }); + OPERATION_RETURN(scope, atom + ? linkedTable.ctiForValue(*unlinkedTable, atom.get()).taggedPtr() + : linkedTable.ctiDefault().taggedPtr()); + } +#endif auto str = string->value(globalObject); OPERATION_RETURN_IF_EXCEPTION(scope, nullptr); - CodeBlock* codeBlock = callFrame->codeBlock(); - const StringJumpTable& linkedTable = codeBlock->dfgStringSwitchJumpTable(tableIndex); OPERATION_RETURN(scope, linkedTable.ctiForValue(*unlinkedTable, str->impl()).taggedPtr()); } +#if USE(BUN_JSC_ADDITIONS) +// Phase C.1 lets fiber words pass speculateStringIdentAndLoadStorage, so either +// argument here may be a bit-1-tagged inline property key rather than a heap +// StringImpl*. Decode to a StringView over the word's embedded bytes before +// handing off to codePointCompare. +static ALWAYS_INLINE std::strong_ordering codePointCompareMaybeInline(StringImpl* a, StringImpl* b) +{ + uintptr_t aWord = reinterpret_cast(a); + uintptr_t bWord = reinterpret_cast(b); + auto viewFor = [](const uintptr_t& word, StringImpl* impl) ALWAYS_INLINE_LAMBDA -> StringView { + if (isInlinePropertyKey(word)) [[unlikely]] { + unsigned len = inlinePropertyKeyLength(word); + const uint8_t* bytes = reinterpret_cast(&word); + if (inlinePropertyKeyIs8Bit(word)) + return StringView { std::span { bytes + 1, len } }; + return StringView { std::span { reinterpret_cast(bytes + 2), len } }; + } + return StringView { impl }; + }; + return codePointCompare(viewFor(aWord, a), viewFor(bWord, b)); +} +#endif + JSC_DEFINE_NOEXCEPT_JIT_OPERATION(operationCompareStringImplLess, uintptr_t, (StringImpl* a, StringImpl* b)) { +#if USE(BUN_JSC_ADDITIONS) + return codePointCompareMaybeInline(a, b) < 0; +#else return codePointCompare(a, b) < 0; +#endif } JSC_DEFINE_NOEXCEPT_JIT_OPERATION(operationCompareStringImplLessEq, uintptr_t, (StringImpl* a, StringImpl* b)) { +#if USE(BUN_JSC_ADDITIONS) + return codePointCompareMaybeInline(a, b) <= 0; +#else return codePointCompare(a, b) <= 0; +#endif } JSC_DEFINE_NOEXCEPT_JIT_OPERATION(operationCompareStringImplGreater, uintptr_t, (StringImpl* a, StringImpl* b)) { +#if USE(BUN_JSC_ADDITIONS) + return codePointCompareMaybeInline(a, b) > 0; +#else return codePointCompare(a, b) > 0; +#endif } JSC_DEFINE_NOEXCEPT_JIT_OPERATION(operationCompareStringImplGreaterEq, uintptr_t, (StringImpl* a, StringImpl* b)) { +#if USE(BUN_JSC_ADDITIONS) + return codePointCompareMaybeInline(a, b) >= 0; +#else return codePointCompare(a, b) >= 0; +#endif } JSC_DEFINE_NOEXCEPT_JIT_OPERATION(operationCompareHeapBigIntLess, uintptr_t, (JSCell* a, JSCell* b)) @@ -4934,6 +5004,37 @@ JSC_DEFINE_JIT_OPERATION(operationHasOwnProperty, size_t, (JSGlobalObject* globa JSValue key = JSValue::decode(encodedKey); if (key.isString()) [[likely]] { +#if USE(BUN_JSC_ADDITIONS) + // DFG/FTL compileHasOwnProperty loads m_fiber and probes the cache at + // uidHash(m_fiber). A fresh inline cell's m_fiber is the fiber word; + // if we call toAtomString() that word is overwritten with the atom + // pointer, and the cache ends up keyed by atom — so every *new* inline + // cell (same content, same fiber word) misses. Key BOTH the object + // lookup and the HasOwnPropertyCache by whatever m_fiber currently + // holds so the next JIT probe (same bits) hits. + GCOwnedDataScope propertyName; + UniquedStringImpl* cacheUid; + UniquedStringImpl* lookupUid; + if (uintptr_t fiber = asString(key)->tryGetCanonicalInlineFiberWord()) { + cacheUid = reinterpret_cast(fiber); + lookupUid = cacheUid; + } else { + propertyName = asString(key)->toAtomString(globalObject); + OPERATION_RETURN_IF_EXCEPTION(scope, false); + cacheUid = propertyName.data; + lookupUid = cacheUid; + if (uintptr_t fiber = Identifier::canonicalFiberWordFor(cacheUid)) + lookupUid = reinterpret_cast(fiber); + } + PropertySlot slot(thisObject, PropertySlot::InternalMethodType::GetOwnProperty); + bool result = thisObject->hasOwnProperty(globalObject, lookupUid, slot); + OPERATION_RETURN_IF_EXCEPTION(scope, false); + + HasOwnPropertyCache* hasOwnPropertyCache = vm.hasOwnPropertyCache(); + ASSERT(hasOwnPropertyCache); + hasOwnPropertyCache->tryAdd(slot, thisObject, cacheUid, result); + OPERATION_RETURN(scope, result); +#else auto propertyName = asString(key)->toAtomString(globalObject); OPERATION_RETURN_IF_EXCEPTION(scope, false); @@ -4945,6 +5046,7 @@ JSC_DEFINE_JIT_OPERATION(operationHasOwnProperty, size_t, (JSGlobalObject* globa ASSERT(hasOwnPropertyCache); hasOwnPropertyCache->tryAdd(slot, thisObject, propertyName.data, result); OPERATION_RETURN(scope, result); +#endif } Identifier propertyName = key.toPropertyKey(globalObject); diff --git a/Source/JavaScriptCore/dfg/DFGSpeculativeJIT.cpp b/Source/JavaScriptCore/dfg/DFGSpeculativeJIT.cpp index 0cfe96f55375a..6e716ca97c6d1 100644 --- a/Source/JavaScriptCore/dfg/DFGSpeculativeJIT.cpp +++ b/Source/JavaScriptCore/dfg/DFGSpeculativeJIT.cpp @@ -2031,6 +2031,42 @@ void SpeculativeJIT::compileStringCodePointAt(Node* node) GPRReg scratch4GPR = scratch4.gpr(); loadPtr(Address(stringGPR, JSString::offsetOfValue()), scratch1GPR); + +#if USE(BUN_JSC_ADDITIONS) + // Same condition FixupPhase used to skip ResolveRope for this node. + SpeculatedType inlinePred = node->child1()->prediction() & SpecString; + if (inlinePred && !(inlinePred & ~SpecStringInline)) { + and32(TrustedImm32(JSString::notStringImplMask), scratch1GPR, scratch2GPR); + speculationCheck(BadType, JSValueSource::unboxedCell(stringGPR), node->child1(), + branch32(NotEqual, scratch2GPR, TrustedImm32(JSString::isInlineInPointer))); + // scratch2 = length, scratch4 = &payload + move(scratch1GPR, scratch2GPR); + urshiftPtr(TrustedImm32(JSString::inlineLengthShift), scratch2GPR); + and32(TrustedImm32(JSString::inlineLengthMask), scratch2GPR); + speculationCheck(Uncountable, JSValueRegs(), nullptr, branch32(AboveOrEqual, indexGPR, scratch2GPR)); + Jump is16 = branchTestPtr(Zero, scratch1GPR, TrustedImm32(JSRopeString::is8BitInPointer)); + // 8-bit: no surrogate handling. + load8(BaseIndex(stringGPR, indexGPR, TimesOne, JSString::offsetOfValue() + 1), scratch1GPR); + Jump done = jump(); + is16.link(this); + addPtr(TrustedImm32(JSString::offsetOfValue() + 2), stringGPR, scratch4GPR); + load16(BaseIndex(scratch4GPR, indexGPR, TimesTwo, 0), scratch1GPR); + add32(TrustedImm32(1), indexGPR, scratch3GPR); + JumpList doneList; + doneList.append(branch32(AboveOrEqual, scratch3GPR, scratch2GPR)); + and32(TrustedImm32(0xfffffc00), scratch1GPR, scratch2GPR); + doneList.append(branch32(NotEqual, scratch2GPR, TrustedImm32(0xd800))); + load16(BaseIndex(scratch4GPR, scratch3GPR, TimesTwo, 0), scratch3GPR); + and32(TrustedImm32(0xfffffc00), scratch3GPR, scratch2GPR); + doneList.append(branch32(NotEqual, scratch2GPR, TrustedImm32(0xdc00))); + lshift32(TrustedImm32(10), scratch1GPR); + getEffectiveAddress(BaseIndex(scratch1GPR, scratch3GPR, TimesOne, -U16_SURROGATE_OFFSET), scratch1GPR); + doneList.link(this); + done.link(this); + strictInt32Result(scratch1GPR, m_currentNode); + return; + } +#endif if (auto stringLength = tryGetConstantStringLength(node->child1())) move(TrustedImm32(*stringLength), scratch2GPR); else @@ -2726,6 +2762,32 @@ void SpeculativeJIT::compileGetCharCodeAt(Node* node) loadPtr(Address(stringReg, JSString::offsetOfValue()), scratchReg); +#if USE(BUN_JSC_ADDITIONS) + // Match the exact condition FixupPhase used to decide whether to skip + // ResolveRope (it reads prediction(), which is fixed before AI runs). + SpeculatedType inlinePred = node->child1()->prediction() & SpecString; + if (inlinePred && !(inlinePred & ~SpecStringInline)) { + // FixupPhase skipped ResolveRope; scratchReg holds the raw fiber. + GPRTemporary scratch2(this); + GPRReg scratch2Reg = scratch2.gpr(); + and32(TrustedImm32(JSString::notStringImplMask), scratchReg, scratch2Reg); + speculationCheck(BadType, JSValueSource::unboxedCell(stringReg), node->child1(), + branch32(NotEqual, scratch2Reg, TrustedImm32(JSString::isInlineInPointer))); + move(scratchReg, scratch2Reg); + urshiftPtr(TrustedImm32(JSString::inlineLengthShift), scratch2Reg); + and32(TrustedImm32(JSString::inlineLengthMask), scratch2Reg); + speculationCheck(Uncountable, JSValueRegs(), nullptr, branch32(AboveOrEqual, indexReg, scratch2Reg)); + Jump is16 = branchTestPtr(Zero, scratchReg, TrustedImm32(JSRopeString::is8BitInPointer)); + load8(BaseIndex(stringReg, indexReg, TimesOne, JSString::offsetOfValue() + 1), scratchReg); + Jump done = jump(); + is16.link(this); + load16(BaseIndex(stringReg, indexReg, TimesTwo, JSString::offsetOfValue() + 2), scratchReg); + done.link(this); + strictInt32Result(scratchReg, m_currentNode); + return; + } +#endif + // unsigned comparison so we can filter out negative indices and indices that are too large if (auto stringLength = tryGetConstantStringLength(node->child1())) speculationCheck(Uncountable, JSValueRegs(), nullptr, branch32(AboveOrEqual, indexReg, TrustedImm32(*stringLength))); @@ -7792,6 +7854,64 @@ void SpeculativeJIT::compileStringEquality( slowCase.append(branchIfRopeStringImpl(implGPR)); }; +#if USE(BUN_JSC_ADDITIONS) + // Literal-vs-inline fast path: encode the (short, 8-bit) literal as an + // inline m_fiber immediate and compare the dynamic side's m_fiber against it. + if (leftAtom != rightAtom) { + const String& constStr = leftAtom ? leftConst : rightConst; + GPRReg dynGPR = leftAtom ? rightGPR : leftGPR; + GPRReg dynTemp = leftAtom ? rightTempGPR : leftTempGPR; + if (constStr.is8Bit() && constStr.length() >= 2 && constStr.length() <= JSString::maxInlineLength8) { + uintptr_t encoded = JSString::encodeInline8(constStr.span8()); + loadPtr(Address(dynGPR, JSString::offsetOfValue()), dynTemp); + Jump notInlineDyn = branchTestPtr(Zero, dynTemp, TrustedImm32(JSString::isInlineInPointer)); + // Dynamic side is inline (bit 1 set, could be rope+substring too: rule that out). + Jump isSubstr = branchTestPtr(NonZero, dynTemp, TrustedImm32(JSString::isRopeInPointer)); + trueCase.append(branchPtr(Equal, dynTemp, TrustedImmPtr(std::bit_cast(encoded)))); + // Dynamic inline, fibers differ: if is8Bit matches, definitely unequal. + falseCase.append(branchTestPtr(NonZero, dynTemp, TrustedImm32(JSRopeString::is8BitInPointer))); + slowCase.append(jump()); + isSubstr.link(this); + notInlineDyn.link(this); + } + } + + // Inline-vs-inline fast path: two inline small strings with matching is8Bit + // are equal iff their m_fiber words are identical (encoding packs + // length|is8Bit|bytes). Mixed 8/16-bit falls through to slowCase via the + // widened rope check in loadImplAndCheckRope. + if (!leftAtom && !rightAtom) { + loadPtr(Address(leftGPR, JSString::offsetOfValue()), leftTempGPR); + loadPtr(Address(rightGPR, JSString::offsetOfValue()), rightTempGPR); + move(leftTempGPR, lengthGPR); + andPtr(rightTempGPR, lengthGPR); + Jump notBothInline = branchTestPtr(Zero, lengthGPR, TrustedImm32(JSString::isInlineInPointer)); + // A single-word compare is only sound for 16-byte inline: len <= 7 + // Latin-1 or len <= 3 UTF-16. lengthGPR still holds AND(left, right); + // is8Bit is set here iff both sides are 8-bit. + Jump both8Bit = branchTestPtr(NonZero, lengthGPR, TrustedImm32(JSRopeString::is8BitInPointer)); + // At least one 16-bit: bail if either length >= 4. + move(leftTempGPR, lengthGPR); + orPtr(rightTempGPR, lengthGPR); + slowCase.append(branchTestPtr(NonZero, lengthGPR, + TrustedImm32((JSString::inlineLengthMask & ~JSString::maxInlineLength16) << JSString::inlineLengthShift))); + Jump widthChecked = jump(); + both8Bit.link(this); + move(leftTempGPR, lengthGPR); + orPtr(rightTempGPR, lengthGPR); + slowCase.append(branchTestPtr(NonZero, lengthGPR, + TrustedImm32((JSString::inlineLengthMask & ~JSString::maxInlineLength8) << JSString::inlineLengthShift))); + widthChecked.link(this); + trueCase.append(branchPtr(Equal, leftTempGPR, rightTempGPR)); + // Both small-inline, fibers differ. Cross-width pair needs the slow compare. + move(leftTempGPR, lengthGPR); + xorPtr(rightTempGPR, lengthGPR); + slowCase.append(branchTestPtr(NonZero, lengthGPR, TrustedImm32(JSRopeString::is8BitInPointer))); + falseCase.append(jump()); + notBothInline.link(this); + } +#endif + auto resolveDataPtr = [&](bool atom, const String& constStr, GPRReg implGPR) { if (atom) { const void* data = constStr.is8Bit() @@ -8532,6 +8652,16 @@ void SpeculativeJIT::compileUnwrapGlobalProxy(Node* node) cellResult(resultGPR, node); } +#if USE(BUN_JSC_ADDITIONS) +bool SpeculativeJIT::isLikelyInlineString(Edge edge) +{ + if (!edge) + return false; + SpeculatedType type = m_state.forNode(edge).m_type & SpecString; + return type && !(type & ~SpecStringInline); +} +#endif + bool SpeculativeJIT::canBeRope(Edge edge) { if (!edge) @@ -8540,8 +8670,13 @@ bool SpeculativeJIT::canBeRope(Edge edge) if (!((m_state.forNode(edge).m_type & SpecString) & ~SpecStringResolved)) return false; if (JSValue value = m_state.forNode(edge).m_value) { +#if USE(BUN_JSC_ADDITIONS) + if (value.isCell() && value.asCell()->type() == StringType && !asString(value)->isRope() && !asString(value)->isInline()) + return false; +#else if (value.isCell() && value.asCell()->type() == StringType && !asString(value)->isRope()) return false; +#endif } // If this value is LazyValue, it will be converted to JSString, and the result must be non-rope string. @@ -8611,6 +8746,20 @@ void SpeculativeJIT::compileGetArrayLength(Node* node) GPRReg tempGPR = temp.gpr(); loadPtr(Address(baseGPR, JSString::offsetOfValue()), tempGPR); +#if USE(BUN_JSC_ADDITIONS) + if (isLikelyInlineString(node->child1())) { + // Profile says this site only sees inline small strings. Speculate on + // that and emit the branch-free decode; OSR exit if we ever see a rope, + // rope-substring, or a resolved StringImpl*. + and32(TrustedImm32(JSString::notStringImplMask), tempGPR, resultGPR); + speculationCheck(BadType, JSValueSource::unboxedCell(baseGPR), node->child1(), + branch32(NotEqual, resultGPR, TrustedImm32(JSString::isInlineInPointer))); + urshiftPtr(TrustedImm32(JSString::inlineLengthShift), tempGPR); + and32(TrustedImm32(JSString::inlineLengthMask), tempGPR, resultGPR); + strictInt32Result(resultGPR, node); + break; + } +#endif Jump isRope; if (canBeRope(node->child1())) isRope = branchIfRopeStringImpl(tempGPR); @@ -8618,7 +8767,17 @@ void SpeculativeJIT::compileGetArrayLength(Node* node) if (isRope.isSet()) { auto done = jump(); isRope.link(this); +#if USE(BUN_JSC_ADDITIONS) + auto realRope = branchTestPtr(NonZero, tempGPR, TrustedImm32(JSString::isRopeInPointer)); + urshiftPtr(TrustedImm32(JSString::inlineLengthShift), tempGPR); + and32(TrustedImm32(JSString::inlineLengthMask), tempGPR, resultGPR); + auto doneInline = jump(); + realRope.link(this); +#endif load32(Address(baseGPR, JSRopeString::offsetOfLength()), resultGPR); +#if USE(BUN_JSC_ADDITIONS) + doneInline.link(this); +#endif done.link(this); } strictInt32Result(resultGPR, node); @@ -12719,9 +12878,18 @@ void SpeculativeJIT::speculateStringIdentAndLoadStorage(Edge edge, GPRReg string if (!needsTypeCheck(edge, SpecStringIdent | ~SpecString)) return; +#if USE(BUN_JSC_ADDITIONS) + if (canBeRope(edge)) + speculationCheck(BadStringType, JSValueSource::unboxedCell(string), edge, branchIfActualRopeStringImpl(storage)); + // Inline small strings are SpecStringInline, not SpecStringIdent; downstream callers pointer-compare against + // heap AtomStringImpl* (compileStringIdentEquality, emitSwitchString), so OSR-exit here to preserve correctness. + speculationCheck(BadStringType, JSValueSource::unboxedCell(string), edge, branchIfInlineStringImpl(storage)); + speculationCheck(BadStringType, JSValueSource::unboxedCell(string), edge, branchTest32(Zero, Address(storage, StringImpl::flagsOffset()), TrustedImm32(StringImpl::flagIsAtom()))); +#else if (canBeRope(edge)) speculationCheck(BadStringType, JSValueSource::unboxedCell(string), edge, branchIfRopeStringImpl(storage)); speculationCheck(BadStringType, JSValueSource::unboxedCell(string), edge, branchTest32(Zero, Address(storage, StringImpl::flagsOffset()), TrustedImm32(StringImpl::flagIsAtom()))); +#endif m_interpreter.filter(edge, SpecStringIdent | ~SpecString); } @@ -13578,6 +13746,10 @@ void SpeculativeJIT::emitSwitchStringOnString(Node* node, SwitchData* data, GPRR if (!isRopeCases.empty()) { isRopeCases.link(this); +#if USE(BUN_JSC_ADDITIONS) + // tempGPR still holds m_fiber here. Inline small strings take the slow operation call. + slowCases.append(branchTestPtr(Zero, tempGPR, TrustedImm32(JSString::isRopeInPointer))); +#endif load32(Address(string, JSRopeString::offsetOfLength()), tempGPR); sub32(TrustedImm32(unlinkedTable.minLength()), tempGPR); branch32(Above, tempGPR, TrustedImm32(unlinkedTable.maxLength() - unlinkedTable.minLength()), data->fallThrough.block); @@ -17674,6 +17846,10 @@ void SpeculativeJIT::compileMakeRope(Node* node) auto done = jump(); isRope.link(this); +#if USE(BUN_JSC_ADDITIONS) + // Inline small-string child: defer to operationMakeRope* (reads past 16-byte cell otherwise). + slowPath.append(branchTestPtr(Zero, scratch2GPR, TrustedImm32(JSString::isRopeInPointer))); +#endif load32(Address(opGPRs[0], JSRopeString::offsetOfFlags()), scratchGPR); load32(Address(opGPRs[0], JSRopeString::offsetOfLength()), allocatorGPR); done.link(this); @@ -17711,6 +17887,10 @@ void SpeculativeJIT::compileMakeRope(Node* node) auto done = jump(); isRope.link(this); +#if USE(BUN_JSC_ADDITIONS) + // Inline small-string child: defer to operationMakeRope*. + slowPath.append(branchTestPtr(Zero, scratch2GPR, TrustedImm32(JSString::isRopeInPointer))); +#endif and32(Address(opGPRs[i], JSRopeString::offsetOfFlags()), scratchGPR); load32(Address(opGPRs[i], JSRopeString::offsetOfLength()), scratch2GPR); outOfMemory.append(branchAdd32(Overflow, scratch2GPR, allocatorGPR)); @@ -17731,6 +17911,10 @@ void SpeculativeJIT::compileMakeRope(Node* node) static_assert(StringImpl::flagIs8Bit() == JSRopeString::is8BitInPointer); and32(TrustedImm32(StringImpl::flagIs8Bit()), scratchGPR); + // No short-result slow-path bail here: the rope cell is already allocated + // above (emitAllocateJSCell reuses allocatorGPR/scratchGPR so the + // allocation cannot be deferred), and routing to operationMakeRope* would + // allocate a second cell and leave this one as garbage. orPtr(opGPRs[0], scratchGPR); orPtr(TrustedImmPtr(JSString::isRopeInPointer), scratchGPR); storePtr(scratchGPR, Address(resultGPR, JSRopeString::offsetOfFiber0())); diff --git a/Source/JavaScriptCore/dfg/DFGSpeculativeJIT.h b/Source/JavaScriptCore/dfg/DFGSpeculativeJIT.h index 3db053a787976..835c5aa9ec314 100644 --- a/Source/JavaScriptCore/dfg/DFGSpeculativeJIT.h +++ b/Source/JavaScriptCore/dfg/DFGSpeculativeJIT.h @@ -571,6 +571,9 @@ class SpeculativeJIT : public JITCompiler { bool isKnownNotOther(Node* node) { return !(m_state.forNode(node).m_type & SpecOther); } bool canBeRope(Edge); +#if USE(BUN_JSC_ADDITIONS) + bool isLikelyInlineString(Edge); +#endif std::optional tryGetConstantStringLength(Edge); UniquedStringImpl* identifierUID(unsigned index) diff --git a/Source/JavaScriptCore/dfg/DFGSpeculativeJIT32_64.cpp b/Source/JavaScriptCore/dfg/DFGSpeculativeJIT32_64.cpp index 06c06e28c75b4..ff5076438fb9d 100644 --- a/Source/JavaScriptCore/dfg/DFGSpeculativeJIT32_64.cpp +++ b/Source/JavaScriptCore/dfg/DFGSpeculativeJIT32_64.cpp @@ -4254,20 +4254,42 @@ void SpeculativeJIT::compile(Node* node) case StringUse: { speculateString(node->child2(), keyRegs.payloadGPR()); loadPtr(Address(keyRegs.payloadGPR(), JSString::offsetOfValue()), implGPR); +#if USE(BUN_JSC_ADDITIONS) + slowPath.append(branchIfActualRopeStringImpl(implGPR)); + { + Jump isInline = branchIfInlineStringImpl(implGPR); + slowPath.append(branchTest32( + Zero, Address(implGPR, StringImpl::flagsOffset()), + TrustedImm32(StringImpl::flagIsAtom()))); + isInline.link(this); + } +#else slowPath.append(branchIfRopeStringImpl(implGPR)); slowPath.append(branchTest32( Zero, Address(implGPR, StringImpl::flagsOffset()), TrustedImm32(StringImpl::flagIsAtom()))); +#endif break; } case UntypedUse: { slowPath.append(branchIfNotCell(keyRegs)); auto isNotString = branchIfNotString(keyRegs.payloadGPR()); loadPtr(Address(keyRegs.payloadGPR(), JSString::offsetOfValue()), implGPR); +#if USE(BUN_JSC_ADDITIONS) + slowPath.append(branchIfActualRopeStringImpl(implGPR)); + { + Jump isInline = branchIfInlineStringImpl(implGPR); + slowPath.append(branchTest32( + Zero, Address(implGPR, StringImpl::flagsOffset()), + TrustedImm32(StringImpl::flagIsAtom()))); + isInline.link(this); + } +#else slowPath.append(branchIfRopeStringImpl(implGPR)); slowPath.append(branchTest32( Zero, Address(implGPR, StringImpl::flagsOffset()), TrustedImm32(StringImpl::flagIsAtom()))); +#endif auto hasUniquedImpl = jump(); isNotString.link(this); @@ -4287,8 +4309,14 @@ void SpeculativeJIT::compile(Node* node) // So we either get super lucky and use zero for the hash and somehow collide with the entity // we're looking for, or we realize we're comparing against another entity, and go to the // slow path anyways. +#if USE(BUN_JSC_ADDITIONS) + // 64-bit mul64/urshift64 are JSVALUE64-only, so compute uidHash(word) + // via the operation unconditionally on 32-bit (matches HasOwnPropertyCache::hash()). + callOperationWithSilentSpill(operationInlinePropertyKeyHash, hashGPR, implGPR); +#else load32(Address(implGPR, UniquedStringImpl::flagsOffset()), hashGPR); urshift32(TrustedImm32(StringImpl::s_flagCount), hashGPR); +#endif load32(Address(objectGPR, JSCell::structureIDOffset()), structureIDGPR); add32(structureIDGPR, hashGPR); and32(TrustedImm32(HasOwnPropertyCache::mask), hashGPR); diff --git a/Source/JavaScriptCore/dfg/DFGSpeculativeJIT64.cpp b/Source/JavaScriptCore/dfg/DFGSpeculativeJIT64.cpp index 7ee9603b5dd71..41f9488b03018 100644 --- a/Source/JavaScriptCore/dfg/DFGSpeculativeJIT64.cpp +++ b/Source/JavaScriptCore/dfg/DFGSpeculativeJIT64.cpp @@ -6053,22 +6053,58 @@ void SpeculativeJIT::compile(Node* node) case StringUse: { speculateString(node->child2(), keyGPR); loadPtr(Address(keyGPR, JSString::offsetOfValue()), implGPR); +#if USE(BUN_JSC_ADDITIONS) + if (canBeRope(node->child2())) + slowPath.append(branchIfActualRopeStringImpl(implGPR)); + if constexpr (enableIdentifierFiberWords) { + Jump isInline = branchIfInlineStringImpl(implGPR); + slowPath.append(branchTest32( + Zero, Address(implGPR, StringImpl::flagsOffset()), + TrustedImm32(StringImpl::flagIsAtom()))); + isInline.link(this); + } else { + // Producer disabled: inline small-string cells must not flow through as a uid. + slowPath.append(branchIfInlineStringImpl(implGPR)); + slowPath.append(branchTest32( + Zero, Address(implGPR, StringImpl::flagsOffset()), + TrustedImm32(StringImpl::flagIsAtom()))); + } +#else if (canBeRope(node->child2())) slowPath.append(branchIfRopeStringImpl(implGPR)); slowPath.append(branchTest32( Zero, Address(implGPR, StringImpl::flagsOffset()), TrustedImm32(StringImpl::flagIsAtom()))); +#endif break; } case UntypedUse: { slowPath.append(branchIfNotCell(JSValueRegs(keyGPR))); auto isNotString = branchIfNotString(keyGPR); loadPtr(Address(keyGPR, JSString::offsetOfValue()), implGPR); +#if USE(BUN_JSC_ADDITIONS) + if (canBeRope(node->child2())) + slowPath.append(branchIfActualRopeStringImpl(implGPR)); + if constexpr (enableIdentifierFiberWords) { + Jump isInline = branchIfInlineStringImpl(implGPR); + slowPath.append(branchTest32( + Zero, Address(implGPR, StringImpl::flagsOffset()), + TrustedImm32(StringImpl::flagIsAtom()))); + isInline.link(this); + } else { + // Producer disabled: inline small-string cells must not flow through as a uid. + slowPath.append(branchIfInlineStringImpl(implGPR)); + slowPath.append(branchTest32( + Zero, Address(implGPR, StringImpl::flagsOffset()), + TrustedImm32(StringImpl::flagIsAtom()))); + } +#else if (canBeRope(node->child2())) slowPath.append(branchIfRopeStringImpl(implGPR)); slowPath.append(branchTest32( Zero, Address(implGPR, StringImpl::flagsOffset()), TrustedImm32(StringImpl::flagIsAtom()))); +#endif auto hasUniquedImpl = jump(); isNotString.link(this); @@ -6088,8 +6124,18 @@ void SpeculativeJIT::compile(Node* node) // So we either get super lucky and use zero for the hash and somehow collide with the entity // we're looking for, or we realize we're comparing against another entity, and go to the // slow path anyways. +#if USE(BUN_JSC_ADDITIONS) + // Branch-free uidHash() = (bits * uidHashMultiplier) >> 32; matches + // HasOwnPropertyCache::hash(). implGPR is the imul src, preserved. + // Option A; not gated on enableIdentifierFiberWords — correct for both reps, + // and JIT stays in lockstep with C++ uidHash() under either flag value. + move(TrustedImm64(static_cast(uidHashMultiplier)), hashGPR); + mul64(implGPR, hashGPR); + urshift64(TrustedImm32(32), hashGPR); +#else load32(Address(implGPR, UniquedStringImpl::flagsOffset()), hashGPR); urshift32(TrustedImm32(StringImpl::s_flagCount), hashGPR); +#endif load32(Address(objectGPR, JSCell::structureIDOffset()), structureIDGPR); add32(structureIDGPR, hashGPR); and32(TrustedImm32(HasOwnPropertyCache::mask), hashGPR); @@ -8400,9 +8446,23 @@ void SpeculativeJIT::compileGetByValWithThisMegamorphic(Node* node) slowCases.append(branchIfNotCell(subscriptRegs)); slowCases.append(branchIfNotString(subscriptRegs.payloadGPR())); loadPtr(Address(subscriptRegs.payloadGPR(), JSString::offsetOfValue()), scratch4GPR); +#if USE(BUN_JSC_ADDITIONS) + if (canBeRope(m_graph.child(node, 2))) + slowCases.append(branchIfActualRopeStringImpl(scratch4GPR)); + if constexpr (enableIdentifierFiberWords) { + Jump isInline = branchIfInlineStringImpl(scratch4GPR); + slowCases.append(branchTest32(Zero, Address(scratch4GPR, StringImpl::flagsOffset()), TrustedImm32(StringImpl::flagIsAtom()))); + isInline.link(this); + } else { + // Producer disabled: inline small-string cells must not flow through as a uid. + slowCases.append(branchIfInlineStringImpl(scratch4GPR)); + slowCases.append(branchTest32(Zero, Address(scratch4GPR, StringImpl::flagsOffset()), TrustedImm32(StringImpl::flagIsAtom()))); + } +#else if (canBeRope(m_graph.child(node, 2))) slowCases.append(branchIfRopeStringImpl(scratch4GPR)); slowCases.append(branchTest32(Zero, Address(scratch4GPR, StringImpl::flagsOffset()), TrustedImm32(StringImpl::flagIsAtom()))); +#endif slowCases.append(loadMegamorphicProperty(vm(), baseGPR, scratch4GPR, nullptr, scratch3GPR, scratch1GPR, scratch2GPR, scratch3GPR)); addSlowPathGenerator(slowPathCall(slowCases, this, operationGetByValWithThisMegamorphicGeneric, scratch3GPR, LinkableConstant::globalObject(*this, node), baseGPR, subscriptRegs, thisValueRegs)); diff --git a/Source/JavaScriptCore/dfg/DFGValidate.cpp b/Source/JavaScriptCore/dfg/DFGValidate.cpp index d700a69368afa..2f928280598d4 100644 --- a/Source/JavaScriptCore/dfg/DFGValidate.cpp +++ b/Source/JavaScriptCore/dfg/DFGValidate.cpp @@ -252,7 +252,11 @@ class Validate { if (node->hasCacheableIdentifier()) { auto* uid = node->cacheableIdentifier().uid(); +#if USE(BUN_JSC_ADDITIONS) + VALIDATE((node), isInlinePropertyKey(uid) || uid->isSymbol() || !parseIndex(*uid)); +#else VALIDATE((node), uid->isSymbol() || !parseIndex(*uid)); +#endif } switch (node->op()) { diff --git a/Source/JavaScriptCore/ftl/FTLLowerDFGToB3.cpp b/Source/JavaScriptCore/ftl/FTLLowerDFGToB3.cpp index 6204924103ad0..62424320e634a 100644 --- a/Source/JavaScriptCore/ftl/FTLLowerDFGToB3.cpp +++ b/Source/JavaScriptCore/ftl/FTLLowerDFGToB3.cpp @@ -4880,9 +4880,17 @@ class LowerDFGToB3 { slowCases.append(jit.branchIfNotCell(subscriptGPR)); slowCases.append(jit.branchIfNotString(subscriptGPR)); jit.loadPtr(CCallHelpers::Address(subscriptGPR, JSString::offsetOfValue()), scratch4GPR); +#if USE(BUN_JSC_ADDITIONS) + if (needsRopeCase) + slowCases.append(jit.branchTestPtr(CCallHelpers::NonZero, scratch4GPR, CCallHelpers::TrustedImm32(JSString::isRopeInPointer))); + auto isInline = jit.branchTestPtr(CCallHelpers::NonZero, scratch4GPR, CCallHelpers::TrustedImm32(JSString::isInlineInPointer)); + slowCases.append(jit.branchTest32(CCallHelpers::Zero, CCallHelpers::Address(scratch4GPR, StringImpl::flagsOffset()), CCallHelpers::TrustedImm32(StringImpl::flagIsAtom()))); + isInline.link(&jit); +#else if (needsRopeCase) slowCases.append(jit.branchIfRopeStringImpl(scratch4GPR)); slowCases.append(jit.branchTest32(CCallHelpers::Zero, CCallHelpers::Address(scratch4GPR, StringImpl::flagsOffset()), CCallHelpers::TrustedImm32(StringImpl::flagIsAtom()))); +#endif slowCases.append(jit.loadMegamorphicProperty(state->vm(), baseGPR, scratch4GPR, nullptr, resultGPR, scratch1GPR, scratch2GPR, scratch3GPR)); CCallHelpers::Label doneForSlow = jit.label(); @@ -6223,22 +6231,61 @@ IGNORE_CLANG_WARNINGS_END return; } +#if USE(BUN_JSC_ADDITIONS) + if (isLikelyInlineString(m_node->child1())) { + LValue fiber = m_out.loadPtr(string, m_heaps.JSString_value); + speculate(BadType, jsValueValue(string), m_node->child1().node(), + m_out.notEqual( + m_out.bitAnd(fiber, m_out.constIntPtr(JSString::notStringImplMask)), + m_out.constIntPtr(JSString::isInlineInPointer))); + setInt32(m_out.bitAnd( + m_out.castToInt32(m_out.lShr(fiber, m_out.constInt32(JSString::inlineLengthShift))), + m_out.constInt32(JSString::inlineLengthMask))); + return; + } +#endif + LBasicBlock ropePath = m_out.newBlock(); LBasicBlock nonRopePath = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); m_out.branch(isRopeString(string, m_node->child1()), rarely(ropePath), usually(nonRopePath)); +#if USE(BUN_JSC_ADDITIONS) + LBasicBlock realRopePath = m_out.newBlock(); + LBasicBlock inlinePath = m_out.newBlock(); + + LBasicBlock lastNext = m_out.appendTo(ropePath, inlinePath); + LValue fiber = m_out.loadPtr(string, m_heaps.JSString_value); + m_out.branch( + m_out.testNonZeroPtr(fiber, m_out.constIntPtr(JSString::isRopeInPointer)), + usually(realRopePath), rarely(inlinePath)); + + m_out.appendTo(inlinePath, realRopePath); + ValueFromBlock inlineLength = m_out.anchor(m_out.bitAnd( + m_out.castToInt32(m_out.lShr(fiber, m_out.constInt32(JSString::inlineLengthShift))), + m_out.constInt32(JSString::inlineLengthMask))); + m_out.jump(continuation); + + m_out.appendTo(realRopePath, nonRopePath); + ValueFromBlock ropeLength = m_out.anchor(m_out.load32NonNegative(string, m_heaps.JSRopeString_length)); + m_out.jump(continuation); +#else LBasicBlock lastNext = m_out.appendTo(ropePath, nonRopePath); ValueFromBlock ropeLength = m_out.anchor(m_out.load32NonNegative(string, m_heaps.JSRopeString_length)); m_out.jump(continuation); +#endif m_out.appendTo(nonRopePath, continuation); ValueFromBlock nonRopeLength = m_out.anchor(m_out.load32NonNegative(m_out.loadPtr(string, m_heaps.JSString_value), m_heaps.StringImpl_length)); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); +#if USE(BUN_JSC_ADDITIONS) + setInt32(m_out.phi(Int32, inlineLength, ropeLength, nonRopeLength)); +#else setInt32(m_out.phi(Int32, ropeLength, nonRopeLength)); +#endif return; } @@ -11698,10 +11745,12 @@ IGNORE_CLANG_WARNINGS_END Allocator allocator = allocatorForConcurrently(vm(), sizeof(JSRopeString), AllocatorForMode::AllocatorIfExists); +#if !USE(BUN_JSC_ADDITIONS) LValue result = allocateCell(m_out.constIntPtr(allocator.localAllocator()), vm().stringStructure.get(), slowPath); // This puts nullptr for the first fiber. It makes visitChildren safe even if this JSRopeString is discarded due to the speculation failure in the following path. m_out.storePtr(m_out.constIntPtr(JSString::isRopeInPointer), result, m_heaps.JSRopeString_fiber0); +#endif auto getFlagsAndLength = [&] (Edge& edge, LValue child) { if (JSString* string = edge->dynamicCastConstant()) { @@ -11718,6 +11767,16 @@ IGNORE_CLANG_WARNINGS_END m_out.branch(isRopeString(child, edge), unsure(ropeCase), unsure(notRopeCase)); LBasicBlock lastNext = m_out.appendTo(ropeCase, notRopeCase); +#if USE(BUN_JSC_ADDITIONS) + // Inline small-string child: the rope-layout loads below would read past the + // 16-byte inline cell, so defer to operationMakeRope* via slowPath. + LBasicBlock realRopeCase = m_out.newBlock(); + LValue childFiber = m_out.loadPtr(child, m_heaps.JSString_value); + m_out.branch( + m_out.testNonZeroPtr(childFiber, m_out.constIntPtr(JSString::isRopeInPointer)), + usually(realRopeCase), rarely(slowPath)); + m_out.appendTo(realRopeCase, notRopeCase); +#endif ValueFromBlock flagsForRope = m_out.anchor(m_out.load32NonNegative(child, m_heaps.JSRopeString_flags)); ValueFromBlock lengthForRope = m_out.anchor(m_out.load32NonNegative(child, m_heaps.JSRopeString_length)); m_out.jump(continuation); @@ -11757,6 +11816,22 @@ IGNORE_CLANG_WARNINGS_END } } +#if USE(BUN_JSC_ADDITIONS) + // Short all-8-bit result: defer to operationMakeRope* so jsString() can + // emit a 16-byte inline cell. Limited to <=7; see DFG compileMakeRope. + { + LBasicBlock notShortInline = m_out.newBlock(); + LValue is8Bit = m_out.testNonZero32(flagsAndLength.flags, m_out.constInt32(StringImpl::flagIs8Bit())); + LValue shortEnough = m_out.belowOrEqual(flagsAndLength.length, m_out.constInt32(JSString::maxInlineLength8)); + m_out.branch(m_out.bitAnd(is8Bit, shortEnough), rarely(slowPath), usually(notShortInline)); + m_out.appendTo(notShortInline); + } + // All slowPath bails (inline child, short result, length overflow) are + // behind us; allocate the rope cell now so it is never wasted. + LValue result = allocateCell(m_out.constIntPtr(allocator.localAllocator()), vm().stringStructure.get(), slowPath); + m_out.storePtr(m_out.constIntPtr(JSString::isRopeInPointer), result, m_heaps.JSRopeString_fiber0); +#endif + m_out.storePtr( m_out.bitOr( m_out.bitOr(kids[0], m_out.constIntPtr(JSString::isRopeInPointer)), @@ -12073,6 +12148,40 @@ IGNORE_CLANG_WARNINGS_END LValue index = lowInt32(m_node->child2()); LValue stringImpl = m_out.loadPtr(base, m_heaps.JSString_value); + +#if USE(BUN_JSC_ADDITIONS) + // Match the exact FixupPhase condition that skipped ResolveRope. + SpeculatedType pred = m_node->child1()->prediction() & SpecString; + if (pred && !(pred & ~SpecStringInline)) { + LBasicBlock inline8 = m_out.newBlock(); + LBasicBlock inline16 = m_out.newBlock(); + LBasicBlock inlineDone = m_out.newBlock(); + speculate(BadType, jsValueValue(base), m_node->child1().node(), + m_out.notEqual( + m_out.bitAnd(stringImpl, m_out.constIntPtr(JSString::notStringImplMask)), + m_out.constIntPtr(JSString::isInlineInPointer))); + LValue inlineLen = m_out.bitAnd( + m_out.castToInt32(m_out.lShr(stringImpl, m_out.constInt32(JSString::inlineLengthShift))), + m_out.constInt32(JSString::inlineLengthMask)); + speculate(Uncountable, noValue(), nullptr, m_out.aboveOrEqual(index, inlineLen)); + m_out.branch( + m_out.testNonZeroPtr(stringImpl, m_out.constIntPtr(JSRopeString::is8BitInPointer)), + unsure(inline8), unsure(inline16)); + LBasicBlock lastNextI = m_out.appendTo(inline8, inline16); + ValueFromBlock ch8 = m_out.anchor(m_out.load8ZeroExt32( + m_out.baseIndex(m_heaps.characters8, base, m_out.zeroExtPtr(index), provenValue(m_node->child2()), JSString::offsetOfValue() + 1))); + m_out.jump(inlineDone); + m_out.appendTo(inline16, inlineDone); + ValueFromBlock ch16 = m_out.anchor(m_out.load16ZeroExt32( + m_out.baseIndex(m_heaps.characters16, base, m_out.zeroExtPtr(index), provenValue(m_node->child2()), JSString::offsetOfValue() + 2))); + m_out.jump(inlineDone); + m_out.appendTo(inlineDone, lastNextI); + ensureStillAliveHere(base); + setInt32(m_out.phi(Int32, ch8, ch16)); + return; + } +#endif + LValue data = m_out.loadPtr(stringImpl, m_heaps.StringImpl_data); if (!m_node->arrayMode().isInBounds()) { @@ -12119,6 +12228,53 @@ IGNORE_CLANG_WARNINGS_END LValue index = lowInt32(m_node->child2()); LValue stringImpl = m_out.loadPtr(base, m_heaps.JSString_value); + +#if USE(BUN_JSC_ADDITIONS) + // Match the FixupPhase condition that skipped ResolveRope. + SpeculatedType pred = m_node->child1()->prediction() & SpecString; + if (pred && !(pred & ~SpecStringInline)) { + speculate(BadType, jsValueValue(base), m_node->child1().node(), + m_out.notEqual( + m_out.bitAnd(stringImpl, m_out.constIntPtr(JSString::notStringImplMask)), + m_out.constIntPtr(JSString::isInlineInPointer))); + LValue inlineLen = m_out.bitAnd( + m_out.castToInt32(m_out.lShr(stringImpl, m_out.constInt32(JSString::inlineLengthShift))), + m_out.constInt32(JSString::inlineLengthMask)); + speculate(Uncountable, noValue(), nullptr, m_out.aboveOrEqual(index, inlineLen)); + LValue data16 = m_out.add(base, m_out.constIntPtr(JSString::offsetOfValue() + 2)); + m_out.branch( + m_out.testNonZeroPtr(stringImpl, m_out.constIntPtr(JSRopeString::is8BitInPointer)), + unsure(is8Bit), unsure(is16Bit)); + + LBasicBlock lastNextI = m_out.appendTo(is8Bit, is16Bit); + ValueFromBlock c8 = m_out.anchor(m_out.load8ZeroExt32( + m_out.baseIndex(m_heaps.characters8, base, m_out.zeroExtPtr(index), provenValue(m_node->child2()), JSString::offsetOfValue() + 1))); + m_out.jump(continuation); + + m_out.appendTo(is16Bit, isLeadSurrogate); + LValue lead = m_out.load16ZeroExt32(m_out.baseIndex(m_heaps.characters16, data16, m_out.zeroExtPtr(index), provenValue(m_node->child2()))); + ValueFromBlock c16 = m_out.anchor(lead); + LValue nextIndex = m_out.add(index, m_out.int32One); + m_out.branch(m_out.aboveOrEqual(nextIndex, inlineLen), unsure(continuation), unsure(isLeadSurrogate)); + + m_out.appendTo(isLeadSurrogate, mayHaveTrailSurrogate); + m_out.branch(m_out.notEqual(m_out.bitAnd(lead, m_out.constInt32(0xfffffc00)), m_out.constInt32(0xd800)), unsure(continuation), unsure(mayHaveTrailSurrogate)); + + m_out.appendTo(mayHaveTrailSurrogate, hasTrailSurrogate); + LValue trail = m_out.load16ZeroExt32(m_out.baseIndex(m_heaps.characters16, data16, m_out.zeroExtPtr(nextIndex))); + m_out.branch(m_out.notEqual(m_out.bitAnd(trail, m_out.constInt32(0xfffffc00)), m_out.constInt32(0xdc00)), unsure(continuation), unsure(hasTrailSurrogate)); + + m_out.appendTo(hasTrailSurrogate, continuation); + ValueFromBlock cSurr = m_out.anchor(m_out.sub(m_out.add(m_out.shl(lead, m_out.constInt32(10)), trail), m_out.constInt32(U16_SURROGATE_OFFSET))); + m_out.jump(continuation); + + m_out.appendTo(continuation, lastNextI); + ensureStillAliveHere(base); + setInt32(m_out.phi(Int32, c8, c16, cSurr)); + return; + } +#endif + LValue length; if (auto stringLength = tryGetConstantStringLength(m_node->child1())) length = m_out.constInt32(*stringLength); @@ -17447,6 +17603,27 @@ IGNORE_CLANG_WARNINGS_END LValue keyAsValue = nullptr; switch (m_node->child2().useKind()) { case StringUse: { +#if USE(BUN_JSC_ADDITIONS) + LBasicBlock isNonEmptyString = m_out.newBlock(); + LBasicBlock isNotInline = m_out.newBlock(); + LBasicBlock isAtomString = m_out.newBlock(); + + keyAsValue = lowString(m_node->child2()); + uniquedStringImpl = m_out.loadPtr(keyAsValue, m_heaps.JSString_value); + m_out.branch(isActualRopeStringImplPtr(uniquedStringImpl), rarely(slowCase), usually(isNonEmptyString)); + + lastNext = m_out.appendTo(isNonEmptyString, isNotInline); + if constexpr (enableIdentifierFiberWords) + m_out.branch(isInlineStringImplPtr(uniquedStringImpl), unsure(isAtomString), unsure(isNotInline)); + else + m_out.branch(isInlineStringImplPtr(uniquedStringImpl), rarely(slowCase), usually(isNotInline)); + + m_out.appendTo(isNotInline, isAtomString); + LValue isNotAtomic = m_out.testIsZero32(m_out.load32(uniquedStringImpl, m_heaps.StringImpl_hashAndFlags), m_out.constInt32(StringImpl::flagIsAtom())); + m_out.branch(isNotAtomic, rarely(slowCase), usually(isAtomString)); + + m_out.appendTo(isAtomString, slowCase); +#else LBasicBlock isNonEmptyString = m_out.newBlock(); LBasicBlock isAtomString = m_out.newBlock(); @@ -17459,6 +17636,7 @@ IGNORE_CLANG_WARNINGS_END m_out.branch(isNotAtomic, rarely(slowCase), usually(isAtomString)); m_out.appendTo(isAtomString, slowCase); +#endif break; } case SymbolUse: { @@ -17472,6 +17650,9 @@ IGNORE_CLANG_WARNINGS_END LBasicBlock isStringCase = m_out.newBlock(); LBasicBlock notStringCase = m_out.newBlock(); LBasicBlock isNonEmptyString = m_out.newBlock(); +#if USE(BUN_JSC_ADDITIONS) + LBasicBlock isNotInline = m_out.newBlock(); +#endif LBasicBlock isSymbolCase = m_out.newBlock(); LBasicBlock hasUniquedStringImpl = m_out.newBlock(); @@ -17481,6 +17662,23 @@ IGNORE_CLANG_WARNINGS_END lastNext = m_out.appendTo(isCellCase, isStringCase); m_out.branch(isString(keyAsValue), unsure(isStringCase), unsure(notStringCase)); +#if USE(BUN_JSC_ADDITIONS) + m_out.appendTo(isStringCase, isNonEmptyString); + LValue implFromString = m_out.loadPtr(keyAsValue, m_heaps.JSString_value); + m_out.branch(isActualRopeStringImplPtr(implFromString), rarely(slowCase), usually(isNonEmptyString)); + + m_out.appendTo(isNonEmptyString, isNotInline); + ValueFromBlock stringInlineResult = m_out.anchor(implFromString); + if constexpr (enableIdentifierFiberWords) + m_out.branch(isInlineStringImplPtr(implFromString), unsure(hasUniquedStringImpl), unsure(isNotInline)); + else + m_out.branch(isInlineStringImplPtr(implFromString), rarely(slowCase), usually(isNotInline)); + + m_out.appendTo(isNotInline, notStringCase); + ValueFromBlock stringResult = m_out.anchor(implFromString); + LValue isNotAtomic = m_out.testIsZero32(m_out.load32(implFromString, m_heaps.StringImpl_hashAndFlags), m_out.constInt32(StringImpl::flagIsAtom())); + m_out.branch(isNotAtomic, rarely(slowCase), usually(hasUniquedStringImpl)); +#else m_out.appendTo(isStringCase, isNonEmptyString); m_out.branch(isNotRopeString(keyAsValue, m_node->child2()), usually(isNonEmptyString), rarely(slowCase)); @@ -17489,6 +17687,7 @@ IGNORE_CLANG_WARNINGS_END ValueFromBlock stringResult = m_out.anchor(implFromString); LValue isNotAtomic = m_out.testIsZero32(m_out.load32(implFromString, m_heaps.StringImpl_hashAndFlags), m_out.constInt32(StringImpl::flagIsAtom())); m_out.branch(isNotAtomic, rarely(slowCase), usually(hasUniquedStringImpl)); +#endif m_out.appendTo(notStringCase, isSymbolCase); m_out.branch(isSymbol(keyAsValue), unsure(isSymbolCase), unsure(slowCase)); @@ -17498,7 +17697,11 @@ IGNORE_CLANG_WARNINGS_END m_out.jump(hasUniquedStringImpl); m_out.appendTo(hasUniquedStringImpl, slowCase); +#if USE(BUN_JSC_ADDITIONS) + uniquedStringImpl = m_out.phi(pointerType(), stringInlineResult, stringResult, symbolResult); +#else uniquedStringImpl = m_out.phi(pointerType(), stringResult, symbolResult); +#endif break; } default: @@ -17513,7 +17716,15 @@ IGNORE_CLANG_WARNINGS_END // So we either get super lucky and use zero for the hash and somehow collide with the entity // we're looking for, or we realize we're comparing against another entity, and go to the // slow path anyways. +#if USE(BUN_JSC_ADDITIONS) + // uniquedStringImpl may be an inline fiber word (bit 1 set) after the atom-check bypass above. + // Branch-free uidHash() = (bits * uidHashMultiplier) >> 32; matches HasOwnPropertyCache::hash(). + // Option A; not gated on enableIdentifierFiberWords — correct for both reps, + // and B3 stays in lockstep with C++ uidHash() under either flag value. + LValue hash = m_out.castToInt32(m_out.lShr(m_out.mul(uniquedStringImpl, m_out.constInt64(uidHashMultiplier)), m_out.constInt32(32))); +#else LValue hash = m_out.lShr(m_out.load32(uniquedStringImpl, m_heaps.StringImpl_hashAndFlags), m_out.constInt32(StringImpl::s_flagCount)); +#endif LValue structureID = m_out.load32(object, m_heaps.JSCell_structureID); LValue index = m_out.add(hash, structureID); @@ -22198,6 +22409,45 @@ IGNORE_CLANG_WARNINGS_END LBasicBlock slowCase = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); +#if USE(BUN_JSC_ADDITIONS) + // Inline-vs-inline: two inline small strings with matching width are + // equal iff their m_fiber words are identical. + std::optional inlineResult; + if (!leftAtom && !rightAtom) { + LBasicBlock inlineSameWidth = m_out.newBlock(); + LBasicBlock inlineDispatch = m_out.newBlock(); + LBasicBlock notBothInline = m_out.newBlock(); + + LValue leftFiber = m_out.loadPtr(leftJSString, m_heaps.JSString_value); + LValue rightFiber = m_out.loadPtr(rightJSString, m_heaps.JSString_value); + LValue xored = m_out.bitXor(leftFiber, rightFiber); + m_out.branch( + m_out.testNonZeroPtr(m_out.bitAnd(leftFiber, rightFiber), m_out.constIntPtr(JSString::isInlineInPointer)), + usually(inlineDispatch), unsure(notBothInline)); + + m_out.appendTo(inlineDispatch, inlineSameWidth); + // Single-word compare is only sound for 16-byte inline: len <= 7 + // Latin-1 or len <= 3 UTF-16. Pick the length mask by the AND'd + // is8Bit bit (set iff both sides are 8-bit). Also bail on cross-width. + LValue ored = m_out.bitOr(leftFiber, rightFiber); + LValue anded = m_out.bitAnd(leftFiber, rightFiber); + LValue both8Bit = m_out.testNonZeroPtr(anded, m_out.constIntPtr(JSRopeString::is8BitInPointer)); + LValue bigMask = m_out.select(both8Bit, + m_out.constIntPtr(static_cast((JSString::inlineLengthMask & ~JSString::maxInlineLength8) << JSString::inlineLengthShift)), + m_out.constIntPtr(static_cast((JSString::inlineLengthMask & ~JSString::maxInlineLength16) << JSString::inlineLengthShift))); + LValue eitherBigOrCrossWidth = m_out.bitOr( + m_out.testNonZeroPtr(ored, bigMask), + m_out.testNonZeroPtr(xored, m_out.constIntPtr(JSRopeString::is8BitInPointer))); + m_out.branch(eitherBigOrCrossWidth, rarely(slowCase), usually(inlineSameWidth)); + + m_out.appendTo(inlineSameWidth, notBothInline); + inlineResult = m_out.anchor(m_out.isZero64(xored)); + m_out.jump(continuation); + + m_out.appendTo(notBothInline, leftReadyCase); + } +#endif + if (leftAtom) m_out.jump(leftReadyCase); else @@ -22348,6 +22598,10 @@ IGNORE_CLANG_WARNINGS_END m_out.jump(continuation); m_out.appendTo(continuation, lastNext); +#if USE(BUN_JSC_ADDITIONS) + if (inlineResult) + return m_out.phi(Int32, trueResult, falseResult, slowResult, *inlineResult); +#endif return m_out.phi(Int32, trueResult, falseResult, slowResult); } @@ -23549,8 +23803,19 @@ IGNORE_CLANG_WARNINGS_END LValue ropeLength; if (auto stringLength = tryGetConstantStringLength(edge)) ropeLength = m_out.constInt32(*stringLength); - else + else { +#if USE(BUN_JSC_ADDITIONS) + // Inline small strings reach here via the widened notStringImplMask test; route them to + // the slow switch to avoid reading the (out-of-bounds) rope length. + LBasicBlock realRopeBlock = m_out.newBlock(); + LValue ropeFiber = m_out.loadPtr(string, m_heaps.JSString_value); + m_out.branch( + m_out.testNonZeroPtr(ropeFiber, m_out.constIntPtr(JSString::isRopeInPointer)), + usually(realRopeBlock), rarely(slowBlock)); + m_out.appendTo(realRopeBlock, slowBlock); +#endif ropeLength = m_out.load32NonNegative(string, m_heaps.JSRopeString_length); + } m_out.branch(m_out.belowOrEqual(m_out.sub(ropeLength, m_out.constInt32(unlinkedTable.minLength())), m_out.constInt32(unlinkedTable.maxLength() - unlinkedTable.minLength())), unsure(slowBlock), unsure(lowBlock(data->fallThrough.block))); m_out.appendTo(slowBlock, lastNext); @@ -25581,6 +25846,16 @@ IGNORE_CLANG_WARNINGS_END m_out.constInt32(StringType)); } +#if USE(BUN_JSC_ADDITIONS) + bool isLikelyInlineString(Edge edge) + { + if (!edge) + return false; + SpeculatedType type = provenType(edge) & SpecString; + return type && !(type & ~SpecStringInline); + } +#endif + bool canBeRope(Edge edge) { if (!edge) @@ -25589,8 +25864,13 @@ IGNORE_CLANG_WARNINGS_END if (!((provenType(edge) & SpecString) & ~SpecStringResolved)) return false; if (JSValue value = provenValue(edge)) { +#if USE(BUN_JSC_ADDITIONS) + if (value.isCell() && value.asCell()->type() == StringType && !asString(value)->isRope() && !asString(value)->isInline()) + return false; +#else if (value.isCell() && value.asCell()->type() == StringType && !asString(value)->isRope()) return false; +#endif } String value = edge->tryGetString(m_graph); if (!value.isNull()) { @@ -25604,7 +25884,7 @@ IGNORE_CLANG_WARNINGS_END { if (!canBeRope(edge)) return m_out.booleanFalse; - return m_out.testNonZeroPtr(m_out.loadPtr(string, m_heaps.JSString_value), m_out.constIntPtr(JSString::isRopeInPointer)); + return m_out.testNonZeroPtr(m_out.loadPtr(string, m_heaps.JSString_value), m_out.constIntPtr(JSString::notStringImplMask)); } std::optional tryGetConstantStringLength(Edge edge) @@ -25621,9 +25901,21 @@ IGNORE_CLANG_WARNINGS_END { if (!canBeRope(edge)) return m_out.booleanTrue; - return m_out.testIsZeroPtr(m_out.loadPtr(string, m_heaps.JSString_value), m_out.constIntPtr(JSString::isRopeInPointer)); + return m_out.testIsZeroPtr(m_out.loadPtr(string, m_heaps.JSString_value), m_out.constIntPtr(JSString::notStringImplMask)); + } + +#if USE(BUN_JSC_ADDITIONS) + LValue isInlineStringImplPtr(LValue stringImpl) + { + return m_out.testNonZeroPtr(stringImpl, m_out.constIntPtr(JSString::isInlineInPointer)); } + LValue isActualRopeStringImplPtr(LValue stringImpl) + { + return m_out.testNonZeroPtr(stringImpl, m_out.constIntPtr(JSString::isRopeInPointer)); + } +#endif + LValue isNotSymbol(LValue cell, SpeculatedType type = SpecFullTop) { if (LValue proven = isProvenValue(type & SpecCell, ~SpecSymbol)) @@ -26113,12 +26405,24 @@ IGNORE_CLANG_WARNINGS_END if (!m_interpreter.needsTypeCheck(edge, SpecStringIdent | ~SpecString)) return; +#if USE(BUN_JSC_ADDITIONS) + speculate(BadStringType, jsValueValue(string), edge.node(), isActualRopeStringImplPtr(stringImpl)); + // Inline small strings are SpecStringInline, not SpecStringIdent; downstream callers pointer-compare against + // heap AtomStringImpl* (CompareStrictEq StringIdentUse, SwitchString), so OSR-exit here to match DFG. + speculate(BadStringType, jsValueValue(string), edge.node(), isInlineStringImplPtr(stringImpl)); + speculate( + BadStringType, jsValueValue(string), edge.node(), + m_out.testIsZero32( + m_out.load32(stringImpl, m_heaps.StringImpl_hashAndFlags), + m_out.constInt32(StringImpl::flagIsAtom()))); +#else speculate(BadStringType, jsValueValue(string), edge.node(), isRopeString(string)); speculate( BadStringType, jsValueValue(string), edge.node(), m_out.testIsZero32( m_out.load32(stringImpl, m_heaps.StringImpl_hashAndFlags), m_out.constInt32(StringImpl::flagIsAtom()))); +#endif m_interpreter.filter(edge, SpecStringIdent | ~SpecString); } diff --git a/Source/JavaScriptCore/heap/BunV8HeapSnapshotBuilder.cpp b/Source/JavaScriptCore/heap/BunV8HeapSnapshotBuilder.cpp index ee6047858d8a6..db1d53553c717 100644 --- a/Source/JavaScriptCore/heap/BunV8HeapSnapshotBuilder.cpp +++ b/Source/JavaScriptCore/heap/BunV8HeapSnapshotBuilder.cpp @@ -22,11 +22,30 @@ #include "DateInstance.h" #include "HeapSnapshotBuilder.h" +#include "InlinePropertyKey.h" #include "RegExpObject.h" #include "HeapSnapshot.h" namespace JSC { +// JSObject::analyzeHeap and JSLexicalEnvironment::analyzeHeap pass +// PropertyTable/SymbolTable keys here; with InlinePropertyKey D.2 those may be +// fiber-word-tagged pointers. This runs from Heap::didFinishCollection where +// the thread AtomStringTable is nulled, so decode to a plain String (heap copy, +// no atom-table lookup) rather than Identifier::string(). +static ALWAYS_INLINE String uidToStringWithoutAtomizing(UniquedStringImpl* uid) +{ + if (isInlinePropertyKey(uid)) [[unlikely]] { + uintptr_t word = inlinePropertyKeyWord(uid); + unsigned len = inlinePropertyKeyLength(word); + const uint8_t* bytes = reinterpret_cast(&word); + if (inlinePropertyKeyIs8Bit(word)) + return String(std::span { bytes + 1, len }); + return String(std::span { reinterpret_cast(bytes + 2), len }); + } + return String(static_cast(uid)); +} + WTF_MAKE_TZONE_ALLOCATED_IMPL(BunV8HeapSnapshotBuilder); BunV8HeapSnapshotBuilder::BunV8HeapSnapshotBuilder(HeapProfiler& profiler) @@ -250,7 +269,7 @@ void BunV8HeapSnapshotBuilder::analyzePropertyNameEdge(JSCell* from, JSCell* to, edge.fromNodeId = getOrCreateNodeId(from); edge.toNodeId = getOrCreateNodeId(to); edge.typeIndex = static_cast(V8EdgeType::Property); - edge.name = WTF::String(propertyName); + edge.name = uidToStringWithoutAtomizing(propertyName); m_edges.append(WTF::move(edge)); } @@ -264,7 +283,7 @@ void BunV8HeapSnapshotBuilder::analyzeVariableNameEdge(JSCell* from, JSCell* to, edge.fromNodeId = getOrCreateNodeId(from); edge.toNodeId = getOrCreateNodeId(to); edge.typeIndex = static_cast(V8EdgeType::Context); - edge.name = String(variableName); + edge.name = uidToStringWithoutAtomizing(variableName); m_edges.append(WTF::move(edge)); } diff --git a/Source/JavaScriptCore/heap/Heap.cpp b/Source/JavaScriptCore/heap/Heap.cpp index 10905dfe305c7..32c6f297ce0df 100644 --- a/Source/JavaScriptCore/heap/Heap.cpp +++ b/Source/JavaScriptCore/heap/Heap.cpp @@ -2347,8 +2347,14 @@ void Heap::finalize() vm().jsonAtomStringCache.clear(); vm().numericStrings.clearOnGarbageCollection(); vm().stringReplaceCache.clear(); +#if USE(BUN_JSC_ADDITIONS) + vm().inlineAtomCache.clear(); +#endif } vm().keyAtomStringCache.clear(); +#if USE(BUN_JSC_ADDITIONS) + vm().inlineStringCache.clear(); +#endif vm().stringSplitCache.clear(); vm().jsonAtomStringCache.clearJSStrings(); diff --git a/Source/JavaScriptCore/heap/Heap.h b/Source/JavaScriptCore/heap/Heap.h index ec349ffe0c17f..d02946b92444d 100644 --- a/Source/JavaScriptCore/heap/Heap.h +++ b/Source/JavaScriptCore/heap/Heap.h @@ -89,6 +89,9 @@ class JSCell; class JSCellButterfly; class JSRopeString; class JSString; +#if USE(BUN_JSC_ADDITIONS) +class JSBigInlineString; +#endif class JSValue; class MachineThreads; class MarkStackArray; @@ -124,6 +127,13 @@ namespace GCClient { class Heap; } +#if USE(BUN_JSC_ADDITIONS) +#define BUN_BIG_INLINE_STRING_SUBSPACE(v) \ + v(bigInlineStringSpace, bigInlineStringHeapCellType, JSBigInlineString) +#else +#define BUN_BIG_INLINE_STRING_SUBSPACE(v) +#endif + #define FOR_EACH_JSC_COMMON_ISO_SUBSPACE(v) \ v(arraySpace, cellHeapCellType, JSArray) \ v(calleeSpace, cellHeapCellType, JSCallee) \ @@ -147,6 +157,7 @@ class Heap; v(regExpSpace, destructibleCellHeapCellType, RegExp) \ v(regExpObjectSpace, cellHeapCellType, RegExpObject) \ v(ropeStringSpace, ropeStringHeapCellType, JSRopeString) \ + BUN_BIG_INLINE_STRING_SUBSPACE(v) \ v(scopedArgumentsSpace, cellHeapCellType, ScopedArguments) \ v(sparseArrayValueMapSpace, destructibleCellHeapCellType, SparseArrayValueMap) \ v(stringSpace, stringHeapCellType, JSString) \ @@ -1077,6 +1088,9 @@ class Heap { IsoHeapCellType nativeStdFunctionHeapCellType; IsoInlinedHeapCellType stringHeapCellType; IsoInlinedHeapCellType ropeStringHeapCellType; +#if USE(BUN_JSC_ADDITIONS) + IsoInlinedHeapCellType bigInlineStringHeapCellType; +#endif IsoHeapCellType weakMapHeapCellType; IsoHeapCellType weakSetHeapCellType; JSDestructibleObjectHeapCellType destructibleObjectHeapCellType; diff --git a/Source/JavaScriptCore/heap/HeapSnapshotBuilder.cpp b/Source/JavaScriptCore/heap/HeapSnapshotBuilder.cpp index 4a4fd40af8850..65400a0539c25 100644 --- a/Source/JavaScriptCore/heap/HeapSnapshotBuilder.cpp +++ b/Source/JavaScriptCore/heap/HeapSnapshotBuilder.cpp @@ -583,7 +583,22 @@ void HeapSnapshotBuilder::dumpToStream(PrintStream& out) if (!firstEdgeName) out.print(','); firstEdgeName = false; +#if USE(BUN_JSC_ADDITIONS) + // edge.name came from JSObject/JSLexicalEnvironment analyzeHeap and may + // hold a fiber-word-tagged uid; String(StringImpl*) would ref-deref it. + if (isInlinePropertyKey(edgeName)) [[unlikely]] { + uintptr_t word = inlinePropertyKeyWord(edgeName); + unsigned len = inlinePropertyKeyLength(word); + const uint8_t* bytes = reinterpret_cast(&word); + if (inlinePropertyKeyIs8Bit(word)) + printJSONString(String(std::span { bytes + 1, len })); + else + printJSONString(String(std::span { reinterpret_cast(bytes + 2), len })); + } else + printJSONString(edgeName); +#else printJSONString(edgeName); +#endif } orderedEdgeNames.clear(); out.print(']'); diff --git a/Source/JavaScriptCore/heap/HeapSnapshotBuilder.h b/Source/JavaScriptCore/heap/HeapSnapshotBuilder.h index f819fc2c4b9d8..083565742ef59 100644 --- a/Source/JavaScriptCore/heap/HeapSnapshotBuilder.h +++ b/Source/JavaScriptCore/heap/HeapSnapshotBuilder.h @@ -26,6 +26,9 @@ #pragma once #include "HeapAnalyzer.h" +#if USE(BUN_JSC_ADDITIONS) +#include "InlinePropertyKey.h" +#endif #include "JSExportMacros.h" #include #include @@ -99,7 +102,11 @@ struct HeapSnapshotEdge { } to; EdgeType type; +#if USE(BUN_JSC_ADDITIONS) + FiberAwareRefPtr name; +#else RefPtr name; +#endif uint32_t index { 0 }; }; diff --git a/Source/JavaScriptCore/inspector/JSInjectedScriptHost.cpp b/Source/JavaScriptCore/inspector/JSInjectedScriptHost.cpp index ee1c95accc678..546bb847de0aa 100644 --- a/Source/JavaScriptCore/inspector/JSInjectedScriptHost.cpp +++ b/Source/JavaScriptCore/inspector/JSInjectedScriptHost.cpp @@ -38,6 +38,9 @@ #include "HeapIterationScope.h" #include "HeapProfiler.h" #include "InjectedScriptHost.h" +#if USE(BUN_JSC_ADDITIONS) +#include "InlinePropertyKey.h" +#endif #include "IterationKind.h" #include "IteratorOperations.h" #include "JSArray.h" @@ -372,7 +375,11 @@ JSValue JSInjectedScriptHost::getOwnPrivatePropertyMethods(JSGlobalObject* globa if (!symbolTable) return; +#if USE(BUN_JSC_ADDITIONS) + Vector> members; +#else Vector, PrivateNameEntry>> members; +#endif { ConcurrentJSLocker locker(symbolTable->m_lock); if (!symbolTable->hasPrivateNames()) @@ -386,9 +393,15 @@ JSValue JSInjectedScriptHost::getOwnPrivatePropertyMethods(JSGlobalObject* globa } } +#if USE(BUN_JSC_ADDITIONS) + std::sort(members.begin(), members.end(), [&vm](const auto& a, const auto& b) { + return codePointCompareLessThan(StringView(Identifier::fromUid(vm, a.first.get()).string()), StringView(Identifier::fromUid(vm, b.first.get()).string())); + }); +#else std::sort(members.begin(), members.end(), [](const auto& a, const auto& b) { return codePointCompareLessThan(StringView(a.first.get()), StringView(b.first.get())); }); +#endif for (const auto& [name, entry] : members) { Identifier memberIdentifier = Identifier::fromUid(vm, name.get()); diff --git a/Source/JavaScriptCore/jit/AssemblyHelpers.cpp b/Source/JavaScriptCore/jit/AssemblyHelpers.cpp index 650e097f54e18..b7ca22aa259f1 100644 --- a/Source/JavaScriptCore/jit/AssemblyHelpers.cpp +++ b/Source/JavaScriptCore/jit/AssemblyHelpers.cpp @@ -509,7 +509,11 @@ AssemblyHelpers::JumpList AssemblyHelpers::loadMegamorphicProperty(VM& vm, GPRRe xor32(scratch2GPR, scratch3GPR); #endif if (uid) +#if USE(BUN_JSC_ADDITIONS) + add32(TrustedImm32(uidHash(uid)), scratch3GPR); +#else add32(TrustedImm32(uid->hash()), scratch3GPR); +#endif else { // Note that we don't test if the hash is zero here. AtomStringImpl's can't have a zero // hash, however, a SymbolImpl may. But, because this is a cache, we don't care. We only @@ -517,9 +521,21 @@ AssemblyHelpers::JumpList AssemblyHelpers::loadMegamorphicProperty(VM& vm, GPRRe // So we either get super lucky and use zero for the hash and somehow collide with the entity // we're looking for, or we realize we're comparing against another entity, and go to the // slow path anyways. +#if USE(BUN_JSC_ADDITIONS) + // Branch-free: uidHash() = (bits * uidHashMultiplier) >> 32 — matches + // MegamorphicCache::primaryHash(). Only scratch2 is consumed (imul64 preserves + // uidGPR), so scratch1 (StructureID) and scratch3 (sid-xor) survive untouched. + // Option A; not gated on enableIdentifierFiberWords — correct for both reps, + // and JIT stays in lockstep with C++ uidHash() under either flag value. + move(TrustedImm64(static_cast(uidHashMultiplier)), scratch2GPR); + mul64(uidGPR, scratch2GPR); + urshift64(TrustedImm32(32), scratch2GPR); + add32(scratch2GPR, scratch3GPR); +#else load32(Address(uidGPR, UniquedStringImpl::flagsOffset()), scratch2GPR); urshift32(TrustedImm32(StringImpl::s_flagCount), scratch2GPR); add32(scratch2GPR, scratch3GPR); +#endif } and32(TrustedImm32(MegamorphicCache::loadCachePrimaryMask), scratch3GPR); @@ -606,7 +622,11 @@ std::tuple AssemblyHelpers #endif if (uid) +#if USE(BUN_JSC_ADDITIONS) + add32(TrustedImm32(uidHash(uid)), scratch3GPR); +#else add32(TrustedImm32(uid->hash()), scratch3GPR); +#endif else { // Note that we don't test if the hash is zero here. AtomStringImpl's can't have a zero // hash, however, a SymbolImpl may. But, because this is a cache, we don't care. We only @@ -614,9 +634,19 @@ std::tuple AssemblyHelpers // So we either get super lucky and use zero for the hash and somehow collide with the entity // we're looking for, or we realize we're comparing against another entity, and go to the // slow path anyways. +#if USE(BUN_JSC_ADDITIONS) + // Branch-free: uidHash() = (bits * uidHashMultiplier) >> 32 — matches + // MegamorphicCache::storeCachePrimaryHash(). Only scratch2 is consumed; scratch1 + // (StructureID) and scratch3 (sid-xor) survive untouched. + move(TrustedImm64(static_cast(uidHashMultiplier)), scratch2GPR); + mul64(uidGPR, scratch2GPR); + urshift64(TrustedImm32(32), scratch2GPR); + add32(scratch2GPR, scratch3GPR); +#else load32(Address(uidGPR, UniquedStringImpl::flagsOffset()), scratch2GPR); urshift32(TrustedImm32(StringImpl::s_flagCount), scratch2GPR); add32(scratch2GPR, scratch3GPR); +#endif } and32(TrustedImm32(MegamorphicCache::storeCachePrimaryMask), scratch3GPR); @@ -701,7 +731,11 @@ AssemblyHelpers::JumpList AssemblyHelpers::hasMegamorphicProperty(VM& vm, GPRReg xor32(scratch2GPR, scratch3GPR); #endif if (uid) +#if USE(BUN_JSC_ADDITIONS) + add32(TrustedImm32(uidHash(uid)), scratch3GPR); +#else add32(TrustedImm32(uid->hash()), scratch3GPR); +#endif else { // Note that we don't test if the hash is zero here. AtomStringImpl's can't have a zero // hash, however, a SymbolImpl may. But, because this is a cache, we don't care. We only @@ -709,9 +743,19 @@ AssemblyHelpers::JumpList AssemblyHelpers::hasMegamorphicProperty(VM& vm, GPRReg // So we either get super lucky and use zero for the hash and somehow collide with the entity // we're looking for, or we realize we're comparing against another entity, and go to the // slow path anyways. +#if USE(BUN_JSC_ADDITIONS) + // Branch-free: uidHash() = (bits * uidHashMultiplier) >> 32 — matches + // MegamorphicCache::hasCachePrimaryHash(). Only scratch2 is consumed; scratch1 + // (StructureID) and scratch3 (sid-xor) survive untouched. + move(TrustedImm64(static_cast(uidHashMultiplier)), scratch2GPR); + mul64(uidGPR, scratch2GPR); + urshift64(TrustedImm32(32), scratch2GPR); + add32(scratch2GPR, scratch3GPR); +#else load32(Address(uidGPR, UniquedStringImpl::flagsOffset()), scratch2GPR); urshift32(TrustedImm32(StringImpl::s_flagCount), scratch2GPR); add32(scratch2GPR, scratch3GPR); +#endif } and32(TrustedImm32(MegamorphicCache::hasCachePrimaryMask), scratch3GPR); @@ -776,9 +820,23 @@ AssemblyHelpers::JumpList AssemblyHelpers::loadCacheableIdentifierImpl(GPRReg pr JumpList slowCases; if (propertyIsString) { loadPtr(Address(propertyGPR, JSString::offsetOfValue()), destGPR); +#if USE(BUN_JSC_ADDITIONS) + if (canBeRope) + slowCases.append(branchIfActualRopeStringImpl(destGPR)); + if constexpr (enableIdentifierFiberWords) { + Jump isInline = branchIfInlineStringImpl(destGPR); + slowCases.append(branchTest32(Zero, Address(destGPR, StringImpl::flagsOffset()), TrustedImm32(StringImpl::flagIsAtom()))); + isInline.link(this); + } else { + // Producer disabled: inline small-string cells must not flow through as a uid. + slowCases.append(branchIfInlineStringImpl(destGPR)); + slowCases.append(branchTest32(Zero, Address(destGPR, StringImpl::flagsOffset()), TrustedImm32(StringImpl::flagIsAtom()))); + } +#else if (canBeRope) slowCases.append(branchIfRopeStringImpl(destGPR)); slowCases.append(branchTest32(Zero, Address(destGPR, StringImpl::flagsOffset()), TrustedImm32(StringImpl::flagIsAtom()))); +#endif } else if (propertyIsSymbol) loadPtr(Address(propertyGPR, Symbol::offsetOfSymbolImpl()), destGPR); else { @@ -790,9 +848,23 @@ AssemblyHelpers::JumpList AssemblyHelpers::loadCacheableIdentifierImpl(GPRReg pr isString.link(this); loadPtr(Address(propertyGPR, JSString::offsetOfValue()), destGPR); +#if USE(BUN_JSC_ADDITIONS) + if (canBeRope) + slowCases.append(branchIfActualRopeStringImpl(destGPR)); + if constexpr (enableIdentifierFiberWords) { + Jump isInline = branchIfInlineStringImpl(destGPR); + slowCases.append(branchTest32(Zero, Address(destGPR, StringImpl::flagsOffset()), TrustedImm32(StringImpl::flagIsAtom()))); + isInline.link(this); + } else { + // Producer disabled: inline small-string cells must not flow through as a uid. + slowCases.append(branchIfInlineStringImpl(destGPR)); + slowCases.append(branchTest32(Zero, Address(destGPR, StringImpl::flagsOffset()), TrustedImm32(StringImpl::flagIsAtom()))); + } +#else if (canBeRope) slowCases.append(branchIfRopeStringImpl(destGPR)); slowCases.append(branchTest32(Zero, Address(destGPR, StringImpl::flagsOffset()), TrustedImm32(StringImpl::flagIsAtom()))); +#endif done.link(this); } diff --git a/Source/JavaScriptCore/jit/AssemblyHelpers.h b/Source/JavaScriptCore/jit/AssemblyHelpers.h index 07fcb39b60f83..426b27ddce3e8 100644 --- a/Source/JavaScriptCore/jit/AssemblyHelpers.h +++ b/Source/JavaScriptCore/jit/AssemblyHelpers.h @@ -1245,16 +1245,31 @@ class AssemblyHelpers : public MacroAssembler { return branchDouble(DoubleEqualAndOrdered, fpr, fpr); } + // "Rope" here means "not a plain StringImpl*" (rope or inline small string). Jump branchIfRopeStringImpl(GPRReg stringImplGPR) { - return branchTestPtr(NonZero, stringImplGPR, TrustedImm32(JSString::isRopeInPointer)); + return branchTestPtr(NonZero, stringImplGPR, TrustedImm32(JSString::notStringImplMask)); } Jump branchIfNotRopeStringImpl(GPRReg stringImplGPR) { - return branchTestPtr(Zero, stringImplGPR, TrustedImm32(JSString::isRopeInPointer)); + return branchTestPtr(Zero, stringImplGPR, TrustedImm32(JSString::notStringImplMask)); + } + +#if USE(BUN_JSC_ADDITIONS) + // Tests only the rope bit (bit 0); inline small strings (bit 1) are not ropes. + Jump branchIfActualRopeStringImpl(GPRReg stringImplGPR) + { + return branchTestPtr(NonZero, stringImplGPR, TrustedImm32(JSString::isRopeInPointer)); } + // Tests only the inline-small-string bit (bit 1). + Jump branchIfInlineStringImpl(GPRReg stringImplGPR) + { + return branchTestPtr(NonZero, stringImplGPR, TrustedImm32(JSString::isInlineInPointer)); + } +#endif + #if USE(JSVALUE64) JumpList branchIfResizableOrGrowableSharedTypedArrayIsOutOfBounds(GPRReg baseGPR, GPRReg scratchGPR, GPRReg scratch2GPR, std::optional); void loadTypedArrayByteLength(GPRReg baseGPR, GPRReg valueGPR, GPRReg scratchGPR, GPRReg scratch2GPR, TypedArrayType); diff --git a/Source/JavaScriptCore/jit/JITOpcodes.cpp b/Source/JavaScriptCore/jit/JITOpcodes.cpp index ba79188fed572..c0a9e6f8be292 100644 --- a/Source/JavaScriptCore/jit/JITOpcodes.cpp +++ b/Source/JavaScriptCore/jit/JITOpcodes.cpp @@ -796,8 +796,15 @@ void JIT::compileOpStrictEq(const JSInstruction* currentInstruction) fallThrough.append(branchIfNotString(stringGPR)); loadPtr(Address(stringGPR, JSString::offsetOfValue()), regT5); +#if USE(BUN_JSC_ADDITIONS) + addSlowCase(branchIfActualRopeStringImpl(regT5)); + // Inline small strings cannot pointer-equal the heap AtomStringImpl* constant below; route to slow path. + addSlowCase(branchIfInlineStringImpl(regT5)); + addSlowCase(branchTest32(Zero, Address(regT5, StringImpl::flagsOffset()), TrustedImm32(StringImpl::flagIsAtom()))); +#else addSlowCase(branchIfRopeStringImpl(regT5)); addSlowCase(branchTest32(Zero, Address(regT5, StringImpl::flagsOffset()), TrustedImm32(StringImpl::flagIsAtom()))); +#endif fallThrough.append(branchPtr(NotEqual, regT5, TrustedImmPtr(string->tryGetValueImpl()))); equals.link(this); @@ -971,8 +978,15 @@ void JIT::compileOpStrictEqJump(const JSInstruction* currentInstruction) fallThrough.append(branchIfNotString(stringGPR)); loadPtr(Address(stringGPR, JSString::offsetOfValue()), regT2); +#if USE(BUN_JSC_ADDITIONS) + addSlowCase(branchIfActualRopeStringImpl(regT2)); + // Inline small strings cannot pointer-equal the heap AtomStringImpl* constant below; route to slow path. + addSlowCase(branchIfInlineStringImpl(regT2)); + addSlowCase(branchTest32(Zero, Address(regT2, StringImpl::flagsOffset()), TrustedImm32(StringImpl::flagIsAtom()))); +#else addSlowCase(branchIfRopeStringImpl(regT2)); addSlowCase(branchTest32(Zero, Address(regT2, StringImpl::flagsOffset()), TrustedImm32(StringImpl::flagIsAtom()))); +#endif addJump(branchPtr(Equal, regT2, TrustedImmPtr(string->tryGetValueImpl())), target); } else { fallThrough.append(branch64(Equal, stringGPR, knownStringGPR)); @@ -980,8 +994,15 @@ void JIT::compileOpStrictEqJump(const JSInstruction* currentInstruction) addJump(branchIfNotString(stringGPR), target); loadPtr(Address(stringGPR, JSString::offsetOfValue()), regT2); +#if USE(BUN_JSC_ADDITIONS) + addSlowCase(branchIfActualRopeStringImpl(regT2)); + // Inline small strings cannot pointer-equal the heap AtomStringImpl* constant below; route to slow path. + addSlowCase(branchIfInlineStringImpl(regT2)); + addSlowCase(branchTest32(Zero, Address(regT2, StringImpl::flagsOffset()), TrustedImm32(StringImpl::flagIsAtom()))); +#else addSlowCase(branchIfRopeStringImpl(regT2)); addSlowCase(branchTest32(Zero, Address(regT2, StringImpl::flagsOffset()), TrustedImm32(StringImpl::flagIsAtom()))); +#endif addJump(branchPtr(NotEqual, regT2, TrustedImmPtr(string->tryGetValueImpl())), target); } fallThrough.link(this); @@ -1367,7 +1388,19 @@ void JIT::emit_op_switch_char(const JSInstruction* currentInstruction) } isRope.link(this); +#if USE(BUN_JSC_ADDITIONS) + // Inline small string: length is in bits 3..7 of regT4. switch_char only matches + // length-1; the only inline length-1 case is a single non-Latin-1 code unit. + auto notInline = branchTestPtr(NonZero, regT4, TrustedImm32(JSString::isRopeInPointer)); + and32(TrustedImm32(JSString::inlineLengthMask << JSString::inlineLengthShift), regT4, regT2); + addJump(branch32(NotEqual, regT2, TrustedImm32(1 << JSString::inlineLengthShift)), defaultOffset); + auto resolveInline = jump(); + notInline.link(this); +#endif addJump(branch32(NotEqual, Address(jsRegT10.payloadGPR(), JSRopeString::offsetOfLength()), TrustedImm32(1)), defaultOffset); +#if USE(BUN_JSC_ADDITIONS) + resolveInline.link(this); +#endif loadGlobalObject(regT2); callOperation(operationResolveRope, regT2, jsRegT10.payloadGPR()); jump().linkTo(dispatch, this); diff --git a/Source/JavaScriptCore/jit/JITOperations.cpp b/Source/JavaScriptCore/jit/JITOperations.cpp index 528855d0b0da8..413e206cdf5aa 100644 --- a/Source/JavaScriptCore/jit/JITOperations.cpp +++ b/Source/JavaScriptCore/jit/JITOperations.cpp @@ -45,6 +45,10 @@ WTF_ALLOW_UNSAFE_BUFFER_USAGE_BEGIN #include "GetterSetter.h" #include "ICStats.h" #include "InlineCacheCompiler.h" +#if USE(BUN_JSC_ADDITIONS) +#include "InlinePropertyKey.h" +#include +#endif #include "Interpreter.h" #include "JIT.h" #include "JITExceptions.h" @@ -929,6 +933,15 @@ static ALWAYS_INLINE JSValue inByValMegamorphic(JSGlobalObject* globalObject, VM JSObject* baseObject = asObject(baseValue); UniquedStringImpl* uid = propertyName.impl(); +#if USE(BUN_JSC_ADDITIONS) + // JIT hasMegamorphicProperty probes with loadPtr(JSString::offsetOfValue()), + // so key the cache by that (atom or fiber word), not the D.4-canonical uid. + UniquedStringImpl* cacheUid = uid; + if (subscript.isString()) { + if (StringImpl* atom = asString(subscript)->tryGetValueImpl()) + cacheUid = static_cast(atom); + } +#endif if (!canUseMegamorphicInById(vm, uid)) [[unlikely]] { dataLogLnIf(verbose, " ", __LINE__); if (propertyCache && propertyCache->considerRepatchingCacheMegamorphic(vm)) { @@ -962,7 +975,11 @@ static ALWAYS_INLINE JSValue inByValMegamorphic(JSGlobalObject* globalObject, VM if (hasProperty) { if (cacheable && slot.isCacheable()) [[likely]] { if (slot.slotBase() == baseObject || !baseObject->structure()->isDictionary()) +#if USE(BUN_JSC_ADDITIONS) + vm.megamorphicCache()->initAsHasHit(baseObject->structureID(), cacheUid); +#else vm.megamorphicCache()->initAsHasHit(baseObject->structureID(), uid); +#endif else { if (baseObject->structure()->hasBeenFlattenedBefore()) [[unlikely]] { dataLogLnIf(verbose, " ", __LINE__); @@ -990,7 +1007,11 @@ static ALWAYS_INLINE JSValue inByValMegamorphic(JSGlobalObject* globalObject, VM if (!prototype.isObject()) { if (cacheable) [[likely]] { if (!baseObject->structure()->isDictionary()) [[likely]] { +#if USE(BUN_JSC_ADDITIONS) + vm.megamorphicCache()->initAsHasMiss(baseObject->structureID(), cacheUid); +#else vm.megamorphicCache()->initAsHasMiss(baseObject->structureID(), uid); +#endif return jsBoolean(false); } if (!baseObject->structure()->hasBeenFlattenedBefore()) [[likely]] @@ -1696,8 +1717,19 @@ static ALWAYS_INLINE void putByVal(JSGlobalObject* globalObject, JSValue baseVal Identifier propertyKey; UniquedStringImpl* uid = nullptr; if (subscript.isString()) { +#if USE(BUN_JSC_ADDITIONS) + if (uintptr_t fiber = asString(subscript)->tryGetCanonicalInlineFiberWord()) + uid = reinterpret_cast(fiber); + else { + propertyName = asString(subscript)->toAtomString(globalObject); + uid = propertyName.data; + if (uintptr_t fiber = Identifier::canonicalFiberWordFor(uid)) + uid = reinterpret_cast(fiber); + } +#else propertyName = asString(subscript)->toAtomString(globalObject); uid = propertyName.data; +#endif } else { propertyKey = subscript.toPropertyKey(globalObject); uid = propertyKey.impl(); @@ -1796,7 +1828,11 @@ static ALWAYS_INLINE void putByValOptimize(JSGlobalObject* globalObject, CodeBlo bool isNonStringPrimitiveKey = false; if (auto propertyName = CacheableIdentifier::getCacheableIdentifier(subscript)) { RETURN_IF_EXCEPTION(scope, void()); +#if USE(BUN_JSC_ADDITIONS) + if (subscript.isSymbol() || !parseIndex(PropertyName(propertyName.data))) +#else if (subscript.isSymbol() || !parseIndex(*propertyName.data)) +#endif identifier = CacheableIdentifier::createFromCell(subscript.asCell()); } else { identifier = nonStringPrimitiveKeyForSubscript(vm, subscript); @@ -1879,7 +1915,11 @@ static ALWAYS_INLINE void directPutByValOptimize(JSGlobalObject* globalObject, C bool isNonStringPrimitiveKey = false; if (auto propertyName = CacheableIdentifier::getCacheableIdentifier(subscript)) { RETURN_IF_EXCEPTION(scope, void()); +#if USE(BUN_JSC_ADDITIONS) + if (subscript.isSymbol() || !parseIndex(PropertyName(propertyName.data))) +#else if (subscript.isSymbol() || !parseIndex(*propertyName.data)) +#endif identifier = CacheableIdentifier::createFromCell(subscript.asCell()); } else { identifier = nonStringPrimitiveKeyForSubscript(vm, subscript); @@ -2075,6 +2115,15 @@ ALWAYS_INLINE static void putByValMegamorphic(JSGlobalObject* globalObject, VM& PutPropertySlot slot(baseValue, isStrict); UniquedStringImpl* uid = propertyName.impl(); +#if USE(BUN_JSC_ADDITIONS) + // JIT storeMegamorphicProperty probes with loadPtr(JSString::offsetOfValue()), + // so key the cache by that (atom or fiber word), not the D.4-canonical uid. + UniquedStringImpl* cacheUid = uid; + if (subscript.isString()) { + if (StringImpl* atom = asString(subscript)->tryGetValueImpl()) + cacheUid = static_cast(atom); + } +#endif if (!canUseMegamorphicPutById(vm, uid) || structure->typeInfo().overridesPut()) [[unlikely]] { if (propertyCache && propertyCache->considerRepatchingCacheMegamorphic(vm)) repatchPutBySlowPathCall(callFrame->codeBlock(), *propertyCache, kind); @@ -2115,7 +2164,11 @@ ALWAYS_INLINE static void putByValMegamorphic(JSGlobalObject* globalObject, VM& if (slot.type() == PutPropertySlot::ExistingProperty) { if (oldStructure == newStructure && slot.cachedOffset() <= MegamorphicCache::maxOffset) [[likely]] { oldStructure->didCachePropertyReplacement(vm, slot.cachedOffset()); // Ensure invalidating watchpoint set. +#if USE(BUN_JSC_ADDITIONS) + vm.megamorphicCache()->initAsReplace(StructureID::encode(oldStructure), cacheUid, slot.cachedOffset()); +#else vm.megamorphicCache()->initAsReplace(StructureID::encode(oldStructure), uid, slot.cachedOffset()); +#endif } return; } @@ -2134,7 +2187,11 @@ ALWAYS_INLINE static void putByValMegamorphic(JSGlobalObject* globalObject, VM& bool reallocating = newStructure->outOfLineCapacity() != oldStructure->outOfLineCapacity(); if (slot.cachedOffset() <= MegamorphicCache::maxOffset) [[likely]] +#if USE(BUN_JSC_ADDITIONS) + vm.megamorphicCache()->initAsTransition(StructureID::encode(oldStructure), StructureID::encode(newStructure), cacheUid, slot.cachedOffset(), reallocating); +#else vm.megamorphicCache()->initAsTransition(StructureID::encode(oldStructure), StructureID::encode(newStructure), uid, slot.cachedOffset(), reallocating); +#endif } JSC_DEFINE_JIT_OPERATION(operationPutByValStrictMegamorphic, void, (EncodedJSValue encodedBaseValue, EncodedJSValue encodedSubscript, EncodedJSValue encodedValue, PropertyInlineCache* propertyCache, ArrayProfile* profile)) @@ -3590,9 +3647,42 @@ ALWAYS_INLINE static JSValue getByVal(JSGlobalObject* globalObject, CallFrame* c auto scope = DECLARE_THROW_SCOPE(vm); if (subscript.isString()) { +#if USE(BUN_JSC_ADDITIONS) + // D.4 coherence: Structure::get()'s m_seenProperties bloom filter and + // PropertyTable are keyed by the canonical fiber word for 2..5-char + // Latin-1 names, so look up with that — not the raw AtomStringImpl*. + // Fast path: grab the word straight from an inline cell's m_fiber and + // skip toAtomString (which would atomize in-place and desync m_fiber + // from the JIT fast-path's last-seen bits). + GCOwnedDataScope propertyName; + UniquedStringImpl* uid; + if (uintptr_t fiber = asString(subscript)->tryGetCanonicalInlineFiberWord()) + uid = reinterpret_cast(fiber); + else { + propertyName = asString(subscript)->toAtomString(globalObject); + RETURN_IF_EXCEPTION(scope, JSValue()); + uid = propertyName.data; + if (uintptr_t fiber = Identifier::canonicalFiberWordFor(uid)) + uid = reinterpret_cast(fiber); + } + + if (baseValue.isCell()) [[likely]] { + Structure& structure = *baseValue.asCell()->structure(); + if (JSCell::canUseFastGetOwnProperty(structure)) { + if (JSValue result = baseValue.asCell()->fastGetOwnProperty(vm, structure, uid)) { + ASSERT(callFrame->bytecodeIndex() != BytecodeIndex(0)); + return result; + } + } + + RELEASE_AND_RETURN(scope, baseValue.get(globalObject, uid)); + } + + ASSERT(callFrame->bytecodeIndex() != BytecodeIndex(0)); + RELEASE_AND_RETURN(scope, baseValue.get(globalObject, uid)); +#else auto propertyName = asString(subscript)->toAtomString(globalObject); RETURN_IF_EXCEPTION(scope, JSValue()); - if (baseValue.isCell()) [[likely]] { Structure& structure = *baseValue.asCell()->structure(); if (JSCell::canUseFastGetOwnProperty(structure)) { @@ -3607,6 +3697,7 @@ ALWAYS_INLINE static JSValue getByVal(JSGlobalObject* globalObject, CallFrame* c ASSERT(callFrame->bytecodeIndex() != BytecodeIndex(0)); RELEASE_AND_RETURN(scope, baseValue.get(globalObject, propertyName.data)); +#endif } else if (std::optional index = subscript.tryGetAsUint32Index()) { uint32_t i = *index; if (isJSString(baseValue)) { @@ -3702,7 +3793,11 @@ JSC_DEFINE_JIT_OPERATION(operationGetByValOptimize, EncodedJSValue, (EncodedJSVa bool isNonStringPrimitiveKey = false; if (auto propertyName = CacheableIdentifier::getCacheableIdentifier(subscript)) { OPERATION_RETURN_IF_EXCEPTION(scope, encodedJSValue()); +#if USE(BUN_JSC_ADDITIONS) + if (subscript.isSymbol() || !parseIndex(PropertyName(propertyName.data))) +#else if (subscript.isSymbol() || !parseIndex(*propertyName.data)) +#endif identifier = CacheableIdentifier::createFromCell(subscript.asCell()); } else { identifier = nonStringPrimitiveKeyForSubscript(vm, subscript); @@ -3734,9 +3829,37 @@ ALWAYS_INLINE static JSValue getByValWithThis(JSGlobalObject* globalObject, Call PropertySlot slot(thisValue, PropertySlot::PropertySlot::InternalMethodType::Get); if (subscript.isString()) { +#if USE(BUN_JSC_ADDITIONS) + // D.4 coherence + perf: grab the canonical fiber word straight from an + // inline cell's m_fiber, skipping the toAtomString()+canonicalFiberWordFor() + // round-trip and leaving m_fiber unmutated for the next JIT probe. + GCOwnedDataScope propertyName; + UniquedStringImpl* uid; + if (uintptr_t fiber = asString(subscript)->tryGetCanonicalInlineFiberWord()) + uid = reinterpret_cast(fiber); + else { + propertyName = asString(subscript)->toAtomString(globalObject); + RETURN_IF_EXCEPTION(scope, { }); + uid = propertyName.data; + if (uintptr_t fiber = Identifier::canonicalFiberWordFor(uid)) + uid = reinterpret_cast(fiber); + } + + if (baseValue.isCell()) [[likely]] { + Structure& structure = *baseValue.asCell()->structure(); + if (JSCell::canUseFastGetOwnProperty(structure)) { + if (JSValue result = baseValue.asCell()->fastGetOwnProperty(vm, structure, uid)) { + ASSERT(callFrame->bytecodeIndex() != BytecodeIndex(0)); + return result; + } + } + } + + ASSERT(callFrame->bytecodeIndex() != BytecodeIndex(0)); + RELEASE_AND_RETURN(scope, baseValue.get(globalObject, uid, slot)); +#else auto propertyName = asString(subscript)->toAtomString(globalObject); RETURN_IF_EXCEPTION(scope, { }); - if (baseValue.isCell()) [[likely]] { Structure& structure = *baseValue.asCell()->structure(); if (JSCell::canUseFastGetOwnProperty(structure)) { @@ -3749,6 +3872,7 @@ ALWAYS_INLINE static JSValue getByValWithThis(JSGlobalObject* globalObject, Call ASSERT(callFrame->bytecodeIndex() != BytecodeIndex(0)); RELEASE_AND_RETURN(scope, baseValue.get(globalObject, propertyName.data, slot)); +#endif } else if (std::optional index = subscript.tryGetAsUint32Index()) { uint32_t i = *index; if (isJSString(baseValue)) { @@ -3813,12 +3937,38 @@ static ALWAYS_INLINE JSValue getByValMegamorphic(JSGlobalObject* globalObject, V GCOwnedDataScope propertyName; Identifier propertyKey; UniquedStringImpl* uid = nullptr; +#if USE(BUN_JSC_ADDITIONS) + // cacheUid: exactly what JSString::m_fiber holds after this block — the JIT + // loadMegamorphicProperty fast path probes with loadPtr(offsetOfValue()), so + // MegamorphicCache::initAsHit/Miss MUST be keyed by that word, not the + // D.4-canonical fiber word. lookupUid: the canonical form for Structure/ + // PropertyTable lookup. For 2-char ASCII substrings (keyAtomStringCache + // atom-backed) cacheUid==atom, lookupUid==fiber; keying the cache by fiber + // here was a 100% miss → 2.6× getByVal regression (Babylon/acorn-wtb). + UniquedStringImpl* cacheUid = nullptr; +#endif if (subscript.isString()) { +#if USE(BUN_JSC_ADDITIONS) + if (uintptr_t fiber = asString(subscript)->tryGetCanonicalInlineFiberWord()) { + cacheUid = reinterpret_cast(fiber); + uid = cacheUid; + } else { + propertyName = asString(subscript)->toAtomString(globalObject); + cacheUid = propertyName.data; + uid = cacheUid; + if (uintptr_t fiber = Identifier::canonicalFiberWordFor(uid)) + uid = reinterpret_cast(fiber); + } +#else propertyName = asString(subscript)->toAtomString(globalObject); uid = propertyName.data; +#endif } else { propertyKey = subscript.toPropertyKey(globalObject); uid = propertyKey.impl(); +#if USE(BUN_JSC_ADDITIONS) + cacheUid = uid; +#endif } RETURN_IF_EXCEPTION(scope, { }); @@ -3830,7 +3980,11 @@ static ALWAYS_INLINE JSValue getByValMegamorphic(JSGlobalObject* globalObject, V RELEASE_AND_RETURN(scope, getByValWithThis(globalObject, callFrame, profile, baseValue, subscript, thisValue)); } +#if USE(BUN_JSC_ADDITIONS) + if (auto index = parseIndex(PropertyName(uid)); index) [[unlikely]] { +#else if (auto index = parseIndex(*uid); index) [[unlikely]] { +#endif if (!baseObject->structure()->isCacheableDictionary() || index.value() >= 100) { if (propertyCache && propertyCache->considerRepatchingCacheMegamorphic(vm)) repatchGetBySlowPathCall(callFrame->codeBlock(), *propertyCache, kind); @@ -3865,7 +4019,11 @@ static ALWAYS_INLINE JSValue getByValMegamorphic(JSGlobalObject* globalObject, V if (hasProperty) { if (cacheable && slot.isCacheableValue() && slot.cachedOffset() <= MegamorphicCache::maxOffset) [[likely]] { if (slot.slotBase() == baseObject || !baseObject->structure()->isDictionary()) +#if USE(BUN_JSC_ADDITIONS) + vm.megamorphicCache()->initAsHit(baseObject->structureID(), cacheUid, slot.slotBase(), slot.cachedOffset(), slot.slotBase() == baseObject); +#else vm.megamorphicCache()->initAsHit(baseObject->structureID(), uid, slot.slotBase(), slot.cachedOffset(), slot.slotBase() == baseObject); +#endif else { if (baseObject->structure()->hasBeenFlattenedBefore()) [[unlikely]] { if (shouldGiveUp && propertyCache && propertyCache->considerRepatchingCacheMegamorphic(vm)) @@ -3889,7 +4047,11 @@ static ALWAYS_INLINE JSValue getByValMegamorphic(JSGlobalObject* globalObject, V if (!prototype.isObject()) { if (cacheable) [[likely]] { if (!baseObject->structure()->isDictionary()) [[likely]] { +#if USE(BUN_JSC_ADDITIONS) + vm.megamorphicCache()->initAsMiss(baseObject->structureID(), cacheUid); +#else vm.megamorphicCache()->initAsMiss(baseObject->structureID(), uid); +#endif return jsUndefined(); } if (!baseObject->structure()->hasBeenFlattenedBefore()) [[likely]] @@ -3992,7 +4154,11 @@ JSC_DEFINE_JIT_OPERATION(operationGetByValWithThisOptimize, EncodedJSValue, (Enc bool isNonStringPrimitiveKey = false; if (auto propertyName = CacheableIdentifier::getCacheableIdentifier(subscript)) { OPERATION_RETURN_IF_EXCEPTION(scope, encodedJSValue()); +#if USE(BUN_JSC_ADDITIONS) + if (subscript.isSymbol() || !parseIndex(PropertyName(propertyName.data))) +#else if (subscript.isSymbol() || !parseIndex(*propertyName.data)) +#endif identifier = CacheableIdentifier::createFromCell(subscript.asCell()); } else { identifier = nonStringPrimitiveKeyForSubscript(vm, subscript); @@ -4628,6 +4794,14 @@ JSC_DEFINE_JIT_OPERATION(operationResolveRopeString, JSString*, (JSGlobalObject* JITOperationPrologueCallFrameTracer tracer(vm, callFrame); auto scope = DECLARE_THROW_SCOPE(vm); +#if USE(BUN_JSC_ADDITIONS) + // compileResolveRope routes inline small strings here via the widened + // notStringImplMask test; those are plain JSStrings, not JSRopeStrings. + if (static_cast(string)->isInline()) { + static_cast(string)->resolveInline(globalObject); + OPERATION_RETURN(scope, static_cast(string)); + } +#endif string->resolveRope(globalObject); OPERATION_RETURN(scope, string); } @@ -4651,6 +4825,15 @@ JSC_DEFINE_JIT_OPERATION(operationSwitchStringWithUnknownKeyType, char*, (JSGlob unsigned length = string->length(); if (length < unlinkedTable.minLength() || length > unlinkedTable.maxLength()) result = linkedTable.ctiDefault().taggedPtr(); +#if USE(BUN_JSC_ADDITIONS) + else if (string->isInline()) { + RefPtr atom = string->is8Bit() + ? AtomStringImpl::lookUp(std::span { string->inlineData8(), length }) + : AtomStringImpl::lookUp(std::span { string->inlineData16(), length }); + result = atom ? linkedTable.ctiForValue(unlinkedTable, atom.get()).taggedPtr() + : linkedTable.ctiDefault().taggedPtr(); + } +#endif else { auto value = string->value(globalObject); OPERATION_RETURN_IF_EXCEPTION(scope, nullptr); @@ -4891,6 +5074,13 @@ JSC_DEFINE_NOEXCEPT_JIT_OPERATION(operationWriteBarrierSlowPath, void, (VM* vmPo vm.writeBarrierSlowPath(cell); } +#if USE(BUN_JSC_ADDITIONS) +JSC_DEFINE_NOEXCEPT_JIT_OPERATION(operationInlinePropertyKeyHash, UCPUStrictInt32, (uintptr_t word)) +{ + return toUCPUStrictInt32(uidHash(reinterpret_cast(word))); +} +#endif + JSC_DEFINE_NOEXCEPT_JIT_OPERATION(operationLookupExceptionHandler, void, (VM* vmPointer)) { VM& vm = *vmPointer; diff --git a/Source/JavaScriptCore/jit/JITOperations.h b/Source/JavaScriptCore/jit/JITOperations.h index 2e90965ee8169..665cdca12409d 100644 --- a/Source/JavaScriptCore/jit/JITOperations.h +++ b/Source/JavaScriptCore/jit/JITOperations.h @@ -445,6 +445,10 @@ JSC_DECLARE_NOEXCEPT_JIT_OPERATION(operationDebuggerWillCallNativeExecutable, vo JSC_DECLARE_NOEXCEPT_JIT_OPERATION(operationProcessTypeProfilerLog, void, (VM*)); JSC_DECLARE_NOEXCEPT_JIT_OPERATION(operationProcessShadowChickenLog, void, (VM*)); +#if USE(BUN_JSC_ADDITIONS) +JSC_DECLARE_NOEXCEPT_JIT_OPERATION(operationInlinePropertyKeyHash, UCPUStrictInt32, (uintptr_t)); +#endif + inline decltype(auto) selectNewFunctionOperation(auto* executable) { auto function = operationNewFunction; diff --git a/Source/JavaScriptCore/jit/ThunkGenerators.cpp b/Source/JavaScriptCore/jit/ThunkGenerators.cpp index b1f9ce7cf283b..bc744456039eb 100644 --- a/Source/JavaScriptCore/jit/ThunkGenerators.cpp +++ b/Source/JavaScriptCore/jit/ThunkGenerators.cpp @@ -694,6 +694,22 @@ MacroAssemblerCodeRef stringEqualThunkGenerator(VM& vm) // Rope: must be substring rope (concat ropes go to slowCase). isRope.link(&jit); +#if USE(BUN_JSC_ADDITIONS) + // Inline small string: decode length/is8Bit from the fiber word and point dataGPR at the + // in-cell character bytes. + auto notInline = jit.branchTestPtr(JIT::NonZero, dataGPR, JIT::TrustedImm32(JSString::isRopeInPointer)); + jit.move(dataGPR, lengthGPR); + jit.urshiftPtr(JIT::TrustedImm32(JSString::inlineLengthShift), lengthGPR); + jit.and32(JIT::TrustedImm32(JSString::inlineLengthMask), lengthGPR); + jit.move(dataGPR, flagsGPR); + jit.and32(JIT::TrustedImm32(StringImpl::flagIs8Bit()), flagsGPR); + jit.addPtr(JIT::TrustedImm32(JSString::offsetOfValue() + 1), jsStringGPR, dataGPR); + auto inlineIs8 = jit.branchTest32(JIT::NonZero, flagsGPR); + jit.addPtr(JIT::TrustedImm32(1), dataGPR); + inlineIs8.link(&jit); + auto inlineDone = jit.jump(); + notInline.link(&jit); +#endif slowCase.append(jit.branchTest64(JIT::Zero, dataGPR, JIT::TrustedImm64(JSRopeString::isSubstringInPointer))); // Load substring base raw (low 48 bits of fiber1) and mask. and64(TrustedImm64) is one @@ -716,6 +732,9 @@ MacroAssemblerCodeRef stringEqualThunkGenerator(VM& vm) jit.addPtr(lengthGPR, dataGPR); jit.load32(JIT::Address(jsStringGPR, JSRopeString::offsetOfLength()), lengthGPR); +#if USE(BUN_JSC_ADDITIONS) + inlineDone.link(&jit); +#endif done.link(&jit); }; diff --git a/Source/JavaScriptCore/jsc.cpp b/Source/JavaScriptCore/jsc.cpp index f800aa458299e..dffc6acdbd81d 100644 --- a/Source/JavaScriptCore/jsc.cpp +++ b/Source/JavaScriptCore/jsc.cpp @@ -601,8 +601,14 @@ class GlobalObject final : public JSGlobalObject { void add(UniquedStringImpl* uid) { auto result = m_names.add(uid); +#if USE(BUN_JSC_ADDITIONS) + // A fiber-word uid is self-owning; nothing to keep alive. + if (result.isNewEntry && !isInlinePropertyKey(uid)) + m_strings.append(uid); +#else if (result.isNewEntry) m_strings.append(uid); +#endif } bool contains(UniquedStringImpl* name) const { return m_names.contains(name); } diff --git a/Source/JavaScriptCore/llint/LLIntSlowPaths.cpp b/Source/JavaScriptCore/llint/LLIntSlowPaths.cpp index bd9805ef1e7d5..8e987d7c66eff 100644 --- a/Source/JavaScriptCore/llint/LLIntSlowPaths.cpp +++ b/Source/JavaScriptCore/llint/LLIntSlowPaths.cpp @@ -1896,6 +1896,16 @@ LLINT_SLOW_PATH_DECL(slow_path_switch_string) unsigned length = string->length(); if (length < unlinkedTable.minLength() || length > unlinkedTable.maxLength()) JUMP_TO(defaultOffset); +#if USE(BUN_JSC_ADDITIONS) + // Inline scrutinee: look up the atom directly from the in-cell bytes. + // Case labels are parser atoms, so a miss means no case matches. + else if (string->isInline()) { + RefPtr atom = string->is8Bit() + ? AtomStringImpl::lookUp(std::span { string->inlineData8(), length }) + : AtomStringImpl::lookUp(std::span { string->inlineData16(), length }); + JUMP_TO(atom ? unlinkedTable.offsetForValue(atom.get()) : defaultOffset); + } +#endif else { auto scrutineeString = string->value(globalObject); LLINT_CHECK_EXCEPTION(); diff --git a/Source/JavaScriptCore/llint/LowLevelInterpreter.asm b/Source/JavaScriptCore/llint/LowLevelInterpreter.asm index d16072c11ed3a..067e3242f18a2 100644 --- a/Source/JavaScriptCore/llint/LowLevelInterpreter.asm +++ b/Source/JavaScriptCore/llint/LowLevelInterpreter.asm @@ -750,6 +750,7 @@ const CallSiteIndex = ArgumentCountIncludingThis + TagOffset # String flags. const isRopeInPointer = constexpr JSString::isRopeInPointer +const notStringImplMask = constexpr JSString::notStringImplMask const HashFlags8BitBuffer = constexpr StringImpl::s_hashFlag8BitBuffer # Copied from PropertyOffset.h diff --git a/Source/JavaScriptCore/llint/LowLevelInterpreter32_64.asm b/Source/JavaScriptCore/llint/LowLevelInterpreter32_64.asm index e4dc554c588c2..fb8d37e143d4f 100644 --- a/Source/JavaScriptCore/llint/LowLevelInterpreter32_64.asm +++ b/Source/JavaScriptCore/llint/LowLevelInterpreter32_64.asm @@ -2228,7 +2228,7 @@ llintOpWithJump(op_switch_char, OpSwitchChar, macro (size, get, jump, dispatch) bineq t1, CellTag, .opSwitchCharFallThrough bbneq JSCell::m_type[t0], StringType, .opSwitchCharFallThrough loadp JSString::m_fiber[t0], t1 - btpnz t1, isRopeInPointer, .opSwitchOnRope + btpnz t1, notStringImplMask, .opSwitchOnRope bineq StringImpl::m_length[t1], 1, .opSwitchCharFallThrough loadi UnlinkedSimpleJumpTable::m_min[t2], t3 @@ -2254,6 +2254,14 @@ llintOpWithJump(op_switch_char, OpSwitchChar, macro (size, get, jump, dispatch) dispatchIndirect(t1) .opSwitchOnRope: + # Inline small string (notStringImplMask set, isRopeInPointer clear): length is in + # bits 3..7 of the fiber in t1; avoid the rope-layout length read below. + btpnz t1, isRopeInPointer, .opSwitchOnRealRope + urshiftp 3, t1 + andp (constexpr JSString::inlineLengthMask), t1 + bineq t1, 1, .opSwitchCharFallThrough + jmp .opSwitchSlow +.opSwitchOnRealRope: bineq JSRopeString::m_compactFibers + JSRopeString::CompactFibers::m_length[t0], 1, .opSwitchCharFallThrough .opSwitchSlow: diff --git a/Source/JavaScriptCore/llint/LowLevelInterpreter64.asm b/Source/JavaScriptCore/llint/LowLevelInterpreter64.asm index 0fffe17155384..0b44bf1038d5d 100644 --- a/Source/JavaScriptCore/llint/LowLevelInterpreter64.asm +++ b/Source/JavaScriptCore/llint/LowLevelInterpreter64.asm @@ -2416,7 +2416,7 @@ llintOpWithJump(op_switch_char, OpSwitchChar, macro (size, get, jump, dispatch) btqnz t1, notCellMask, .opSwitchCharFallThrough bbneq JSCell::m_type[t1], StringType, .opSwitchCharFallThrough loadp JSString::m_fiber[t1], t0 - btpnz t0, isRopeInPointer, .opSwitchOnRope + btpnz t0, notStringImplMask, .opSwitchOnRope bineq StringImpl::m_length[t0], 1, .opSwitchCharFallThrough loadi UnlinkedSimpleJumpTable::m_min[t2], t3 @@ -2442,6 +2442,14 @@ llintOpWithJump(op_switch_char, OpSwitchChar, macro (size, get, jump, dispatch) dispatchIndirect(t1) .opSwitchOnRope: + # Inline small string (notStringImplMask set, isRopeInPointer clear): length is in + # bits 3..7 of the fiber in t0; avoid the rope-layout length read below. + btpnz t0, isRopeInPointer, .opSwitchOnRealRope + urshiftp 3, t0 + andp (constexpr JSString::inlineLengthMask), t0 + bineq t0, 1, .opSwitchCharFallThrough + jmp .opSwitchSlow +.opSwitchOnRealRope: bineq JSRopeString::m_compactFibers + JSRopeString::CompactFibers::m_length[t1], 1, .opSwitchCharFallThrough .opSwitchSlow: diff --git a/Source/JavaScriptCore/lol/LOLJIT.cpp b/Source/JavaScriptCore/lol/LOLJIT.cpp index c129bd258f3e5..b10f1b533b4f2 100644 --- a/Source/JavaScriptCore/lol/LOLJIT.cpp +++ b/Source/JavaScriptCore/lol/LOLJIT.cpp @@ -2450,7 +2450,19 @@ void LOLJIT::emit_op_switch_char(const JSInstruction* currentInstruction) } isRope.link(this); +#if USE(BUN_JSC_ADDITIONS) + // Inline small string: length is in bits 3..7 of regT4. switch_char only matches + // length-1; the only inline length-1 case is a single non-Latin-1 code unit. + auto notInline = branchTestPtr(NonZero, regT4, TrustedImm32(JSString::isRopeInPointer)); + and32(TrustedImm32(JSString::inlineLengthMask << JSString::inlineLengthShift), regT4, s_scratch); + addJump(branch32(NotEqual, s_scratch, TrustedImm32(1 << JSString::inlineLengthShift)), defaultOffset); + auto resolveInline = jump(); + notInline.link(this); +#endif addJump(branch32(NotEqual, Address(scrutineeRegs.payloadGPR(), JSRopeString::offsetOfLength()), TrustedImm32(1)), defaultOffset); +#if USE(BUN_JSC_ADDITIONS) + resolveInline.link(this); +#endif loadGlobalObject(s_scratch); callOperation(operationResolveRope, s_scratch, scrutineeRegs.payloadGPR()); jump().linkTo(dispatch, this); diff --git a/Source/JavaScriptCore/parser/ModuleAnalyzer.cpp b/Source/JavaScriptCore/parser/ModuleAnalyzer.cpp index e863c98ad8fe1..894dcabe75d32 100644 --- a/Source/JavaScriptCore/parser/ModuleAnalyzer.cpp +++ b/Source/JavaScriptCore/parser/ModuleAnalyzer.cpp @@ -47,7 +47,11 @@ void ModuleAnalyzer::appendRequestedModule(const Identifier& specifier, RefPtrappendRequestedModule(specifier, WTF::move(attributes), phase); } +#if USE(BUN_JSC_ADDITIONS) +void ModuleAnalyzer::exportVariable(ModuleProgramNode& moduleProgramNode, const FiberAwareRefPtr& localName, const VariableEnvironmentEntry& variable) +#else void ModuleAnalyzer::exportVariable(ModuleProgramNode& moduleProgramNode, const RefPtr& localName, const VariableEnvironmentEntry& variable) +#endif { // In the parser, we already marked the variables as Exported and Imported. // By leveraging this information, we collect the information that is needed diff --git a/Source/JavaScriptCore/parser/ModuleAnalyzer.h b/Source/JavaScriptCore/parser/ModuleAnalyzer.h index c595386d98c4a..298648b496c32 100644 --- a/Source/JavaScriptCore/parser/ModuleAnalyzer.h +++ b/Source/JavaScriptCore/parser/ModuleAnalyzer.h @@ -53,7 +53,11 @@ class ModuleAnalyzer { void fail(std::tuple&& errorMessage) { m_errorMessage = errorMessage; } private: +#if USE(BUN_JSC_ADDITIONS) + void exportVariable(ModuleProgramNode&, const FiberAwareRefPtr&, const VariableEnvironmentEntry&); +#else void exportVariable(ModuleProgramNode&, const RefPtr&, const VariableEnvironmentEntry&); +#endif VM& m_vm; JSModuleRecord* m_moduleRecord; diff --git a/Source/JavaScriptCore/parser/ModuleScopeData.h b/Source/JavaScriptCore/parser/ModuleScopeData.h index 2d4648afefc8f..d88705c27f368 100644 --- a/Source/JavaScriptCore/parser/ModuleScopeData.h +++ b/Source/JavaScriptCore/parser/ModuleScopeData.h @@ -36,7 +36,12 @@ class ModuleScopeData : public RefCounted { WTF_MAKE_NONCOPYABLE(ModuleScopeData); WTF_MAKE_TZONE_ALLOCATED(ModuleScopeData); public: +#if USE(BUN_JSC_ADDITIONS) + // Keys/values may be inline fiber words (not real StringImpl*); FiberAwareRefPtr skips ref()/deref() on those. + typedef UncheckedKeyHashMap, IdentifierRepHash, HashTraits> IdentifierAliasMap; +#else typedef UncheckedKeyHashMap, Vector>, IdentifierRepHash, HashTraits>> IdentifierAliasMap; +#endif static Ref create() { return adoptRef(*new ModuleScopeData); } @@ -49,7 +54,11 @@ class ModuleScopeData : public RefCounted { void exportBinding(const Identifier& localName, const Identifier& exportedName) { +#if USE(BUN_JSC_ADDITIONS) + m_exportedBindings.add(localName.impl(), Vector()).iterator->value.append(exportedName.impl()); +#else m_exportedBindings.add(localName.impl(), Vector>()).iterator->value.append(exportedName.impl()); +#endif } void exportBinding(const Identifier& localName) diff --git a/Source/JavaScriptCore/parser/Parser.cpp b/Source/JavaScriptCore/parser/Parser.cpp index a71bce8e45892..f3e88a476afdd 100644 --- a/Source/JavaScriptCore/parser/Parser.cpp +++ b/Source/JavaScriptCore/parser/Parser.cpp @@ -209,7 +209,20 @@ void JSToken::dump(PrintStream& out) const static ALWAYS_INLINE bool NODELETE isPrivateFieldName(UniquedStringImpl* uid) { +#if USE(BUN_JSC_ADDITIONS) + if (isInlinePropertyKey(uid)) { + uintptr_t word = reinterpret_cast(uid); + if (!inlinePropertyKeyLength(word)) + return false; + const uint8_t* bytes = reinterpret_cast(&word); + if (inlinePropertyKeyIs8Bit(word)) + return bytes[1] == '#'; + return reinterpret_cast(bytes + 2)[0] == '#'; + } + return uid->length() && uid->at(0) == '#'; +#else return uid->length() && uid->at(0) == '#'; +#endif } template @@ -1113,7 +1126,11 @@ template TreeDestructuringPattern Parser::createB { ASSERT(!name.isNull()); +#if USE(BUN_JSC_ADDITIONS) + ASSERT(isInlinePropertyKey(name.impl()) || name.impl()->isAtom() || name.impl()->isSymbol()); +#else ASSERT(name.impl()->isAtom() || name.impl()->isSymbol()); +#endif switch (kind) { case DestructuringKind::DestructureToVariables: { diff --git a/Source/JavaScriptCore/parser/SourceProviderCacheItem.h b/Source/JavaScriptCore/parser/SourceProviderCacheItem.h index 3049c07837703..301ca18b89cdd 100644 --- a/Source/JavaScriptCore/parser/SourceProviderCacheItem.h +++ b/Source/JavaScriptCore/parser/SourceProviderCacheItem.h @@ -26,6 +26,9 @@ #pragma once #include "ImplementationVisibility.h" +#if USE(BUN_JSC_ADDITIONS) +#include "InlinePropertyKey.h" +#endif #include "ParserModes.h" #include "ParserTokens.h" #include @@ -119,8 +122,13 @@ class SourceProviderCacheItem { inline SourceProviderCacheItem::~SourceProviderCacheItem() { +#if USE(BUN_JSC_ADDITIONS) + for (unsigned i = 0; i < usedVariablesCount; ++i) + uidDeref(m_variables[i].get()); +#else for (unsigned i = 0; i < usedVariablesCount; ++i) m_variables[i]->deref(); +#endif } inline std::unique_ptr SourceProviderCacheItem::create(const SourceProviderCacheItemCreationParameters& parameters) @@ -159,7 +167,11 @@ inline SourceProviderCacheItem::SourceProviderCacheItem(const SourceProviderCach ASSERT(expectedSuperBinding == static_cast(parameters.expectedSuperBinding)); for (unsigned i = 0; i < usedVariablesCount; ++i) { auto* pointer = parameters.usedVariables[i]; +#if USE(BUN_JSC_ADDITIONS) + uidRef(pointer); +#else pointer->ref(); +#endif m_variables[i] = pointer; } } diff --git a/Source/JavaScriptCore/parser/VariableEnvironment.cpp b/Source/JavaScriptCore/parser/VariableEnvironment.cpp index 902b8f938faeb..f414ba1b527a3 100644 --- a/Source/JavaScriptCore/parser/VariableEnvironment.cpp +++ b/Source/JavaScriptCore/parser/VariableEnvironment.cpp @@ -118,7 +118,7 @@ void VariableEnvironment::markVariableAsExported(const UniquedStringImpl* identi findResult->value.setIsExported(); } -VariableEnvironment::Map::AddResult VariableEnvironment::declarePrivateField(const RefPtr& identifier) +VariableEnvironment::Map::AddResult VariableEnvironment::declarePrivateField(const VERefPtr& identifier) { getOrAddPrivateName(identifier.get()); auto entry = VariableEnvironmentEntry(); @@ -128,7 +128,7 @@ VariableEnvironment::Map::AddResult VariableEnvironment::declarePrivateField(con return m_map.add(identifier, entry); } -VariableEnvironment::PrivateDeclarationResult VariableEnvironment::declarePrivateAccessor(const RefPtr& identifier, PrivateNameEntry accessorTraits) +VariableEnvironment::PrivateDeclarationResult VariableEnvironment::declarePrivateAccessor(const VERefPtr& identifier, PrivateNameEntry accessorTraits) { if (!m_rareData) m_rareData = WTF::makeUnique(); @@ -176,17 +176,17 @@ VariableEnvironment::PrivateDeclarationResult VariableEnvironment::declarePrivat return PrivateDeclarationResult::Success; } -VariableEnvironment::PrivateDeclarationResult VariableEnvironment::declarePrivateSetter(const RefPtr& identifier, PrivateNameEntry::Traits modifierTraits) +VariableEnvironment::PrivateDeclarationResult VariableEnvironment::declarePrivateSetter(const VERefPtr& identifier, PrivateNameEntry::Traits modifierTraits) { return declarePrivateAccessor(identifier, PrivateNameEntry(PrivateNameEntry::Traits::IsSetter | modifierTraits)); } -VariableEnvironment::PrivateDeclarationResult VariableEnvironment::declarePrivateGetter(const RefPtr& identifier, PrivateNameEntry::Traits modifierTraits) +VariableEnvironment::PrivateDeclarationResult VariableEnvironment::declarePrivateGetter(const VERefPtr& identifier, PrivateNameEntry::Traits modifierTraits) { return declarePrivateAccessor(identifier, PrivateNameEntry(PrivateNameEntry::Traits::IsGetter | modifierTraits)); } -bool VariableEnvironment::declarePrivateMethod(const RefPtr& identifier, PrivateNameEntry::Traits addionalTraits) +bool VariableEnvironment::declarePrivateMethod(const VERefPtr& identifier, PrivateNameEntry::Traits addionalTraits) { if (!m_rareData) m_rareData = makeUnique(); @@ -226,10 +226,17 @@ void CompactTDZEnvironment::sortCompact(Compact& compact) CompactTDZEnvironment::CompactTDZEnvironment(const TDZEnvironment& env) { m_hash = 0; // Note: XOR is commutative so order doesn't matter here. +#if USE(BUN_JSC_ADDITIONS) + Compact variables = WTF::map(env, [this](auto& key) -> FiberAwarePackedRefPtr { + m_hash ^= uidHash(key.get()); + return key.get(); + }); +#else Compact variables = WTF::map(env, [this](auto& key) -> PackedRefPtr { m_hash ^= key->hash(); return key.get(); }); +#endif sortCompact(variables); m_variables = WTF::move(variables); diff --git a/Source/JavaScriptCore/parser/VariableEnvironment.h b/Source/JavaScriptCore/parser/VariableEnvironment.h index 1dda303c94685..523579051b863 100644 --- a/Source/JavaScriptCore/parser/VariableEnvironment.h +++ b/Source/JavaScriptCore/parser/VariableEnvironment.h @@ -142,7 +142,13 @@ struct PrivateNameEntryHashTraits : HashTraits { static constexpr bool needsDestruction = false; }; +#if USE(BUN_JSC_ADDITIONS) +typedef UncheckedKeyHashMap, PrivateNameEntryHashTraits> PrivateNameEnvironment; +using VERefPtr = FiberAwareRefPtr; +#else typedef UncheckedKeyHashMap, PrivateNameEntry, IdentifierRepHash, HashTraits>, PrivateNameEntryHashTraits> PrivateNameEnvironment; +using VERefPtr = RefPtr; +#endif class VariableEnvironment { WTF_MAKE_TZONE_ALLOCATED(VariableEnvironment); @@ -151,7 +157,11 @@ class VariableEnvironment { static constexpr unsigned inlineMapCapacity = 9; private: +#if USE(BUN_JSC_ADDITIONS) + typedef InlineMap, VariableEnvironmentEntryHashTraits> Map; +#else typedef InlineMap, VariableEnvironmentEntry, inlineMapCapacity, IdentifierRepHash, HashTraits>, VariableEnvironmentEntryHashTraits> Map; +#endif public: @@ -171,13 +181,18 @@ class VariableEnvironment { ALWAYS_INLINE Map::iterator end() { return m_map.end(); } ALWAYS_INLINE Map::const_iterator begin() const { return m_map.begin(); } ALWAYS_INLINE Map::const_iterator end() const { return m_map.end(); } - ALWAYS_INLINE Map::AddResult add(const RefPtr& identifier) { return m_map.add(identifier, VariableEnvironmentEntry()); } +#if USE(BUN_JSC_ADDITIONS) + ALWAYS_INLINE Map::AddResult add(UniquedStringImpl* identifier) { return m_map.add(FiberAwareRefPtr(identifier), VariableEnvironmentEntry()); } + ALWAYS_INLINE Map::AddResult add(const FiberAwareRefPtr& identifier) { return m_map.add(identifier, VariableEnvironmentEntry()); } +#else + ALWAYS_INLINE Map::AddResult add(const VERefPtr& identifier) { return m_map.add(identifier, VariableEnvironmentEntry()); } +#endif ALWAYS_INLINE Map::AddResult add(const Identifier& identifier) { return add(identifier.impl()); } // Defined in VariableEnvironmentInlines.h. PrivateNameEnvironment::AddResult addPrivateName(const Identifier& identifier); // Defined in VariableEnvironmentInlines.h. - PrivateNameEnvironment::AddResult addPrivateName(const RefPtr& identifier); + PrivateNameEnvironment::AddResult addPrivateName(const VERefPtr& identifier); ALWAYS_INLINE unsigned size() const { return m_map.size() + privateNamesSize(); } ALWAYS_INLINE unsigned mapSize() const { return m_map.size(); } @@ -223,7 +238,7 @@ class VariableEnvironment { // Defined in VariableEnvironmentInlines.h. bool declarePrivateMethod(const Identifier& identifier); - bool declarePrivateMethod(const RefPtr& identifier, PrivateNameEntry::Traits addionalTraits = PrivateNameEntry::Traits::None); + bool declarePrivateMethod(const VERefPtr& identifier, PrivateNameEntry::Traits addionalTraits = PrivateNameEntry::Traits::None); enum class PrivateDeclarationResult { Success, @@ -231,7 +246,7 @@ class VariableEnvironment { InvalidStaticNonStatic }; - PrivateDeclarationResult declarePrivateAccessor(const RefPtr&, PrivateNameEntry accessorTraits); + PrivateDeclarationResult declarePrivateAccessor(const VERefPtr&, PrivateNameEntry accessorTraits); // Defined in VariableEnvironmentInlines.h. bool declareStaticPrivateMethod(const Identifier& identifier); @@ -240,15 +255,15 @@ class VariableEnvironment { PrivateDeclarationResult declarePrivateSetter(const Identifier& identifier); // Defined in VariableEnvironmentInlines.h. PrivateDeclarationResult declareStaticPrivateSetter(const Identifier& identifier); - PrivateDeclarationResult declarePrivateSetter(const RefPtr& identifier, PrivateNameEntry::Traits modifierTraits = PrivateNameEntry::Traits::None); + PrivateDeclarationResult declarePrivateSetter(const VERefPtr& identifier, PrivateNameEntry::Traits modifierTraits = PrivateNameEntry::Traits::None); // Defined in VariableEnvironmentInlines.h. PrivateDeclarationResult declarePrivateGetter(const Identifier& identifier); // Defined in VariableEnvironmentInlines.h. PrivateDeclarationResult declareStaticPrivateGetter(const Identifier& identifier); - PrivateDeclarationResult declarePrivateGetter(const RefPtr& identifier, PrivateNameEntry::Traits modifierTraits = PrivateNameEntry::Traits::None); + PrivateDeclarationResult declarePrivateGetter(const VERefPtr& identifier, PrivateNameEntry::Traits modifierTraits = PrivateNameEntry::Traits::None); - Map::AddResult declarePrivateField(const RefPtr&); + Map::AddResult declarePrivateField(const VERefPtr&); ALWAYS_INLINE PrivateNamesRange privateNames() const { @@ -342,7 +357,11 @@ class VariableEnvironment { std::unique_ptr m_rareData; }; +#if USE(BUN_JSC_ADDITIONS) +using TDZEnvironment = UncheckedKeyHashSet; +#else using TDZEnvironment = UncheckedKeyHashSet, IdentifierRepHash>; +#endif class CompactTDZEnvironment { WTF_MAKE_TZONE_ALLOCATED(CompactTDZEnvironment); @@ -350,7 +369,11 @@ class CompactTDZEnvironment { friend class CachedCompactTDZEnvironment; +#if USE(BUN_JSC_ADDITIONS) + using Compact = Vector; +#else using Compact = Vector>; +#endif using Inflated = TDZEnvironment; using Variables = Variant; diff --git a/Source/JavaScriptCore/parser/VariableEnvironmentInlines.h b/Source/JavaScriptCore/parser/VariableEnvironmentInlines.h index 80d81f8c0cc82..557d623a3fdb3 100644 --- a/Source/JavaScriptCore/parser/VariableEnvironmentInlines.h +++ b/Source/JavaScriptCore/parser/VariableEnvironmentInlines.h @@ -100,7 +100,7 @@ ALWAYS_INLINE PrivateNameEnvironment::AddResult VariableEnvironment::addPrivateN return addPrivateName(identifier.impl()); } -ALWAYS_INLINE PrivateNameEnvironment::AddResult VariableEnvironment::addPrivateName(const RefPtr& identifier) +ALWAYS_INLINE PrivateNameEnvironment::AddResult VariableEnvironment::addPrivateName(const VERefPtr& identifier) { if (!m_rareData) m_rareData = makeUnique(); diff --git a/Source/JavaScriptCore/runtime/AbstractModuleRecord.cpp b/Source/JavaScriptCore/runtime/AbstractModuleRecord.cpp index 78f24884d12bd..7c992f600ff52 100644 --- a/Source/JavaScriptCore/runtime/AbstractModuleRecord.cpp +++ b/Source/JavaScriptCore/runtime/AbstractModuleRecord.cpp @@ -318,7 +318,11 @@ struct AbstractModuleRecord::ResolveQuery { // The module record is not marked from the GC. But these records are reachable from the JSGlobalObject. // So we don't care the reachability to this record. AbstractModuleRecord* moduleRecord; +#if USE(BUN_JSC_ADDITIONS) + FiberAwareRefPtr exportName; +#else RefPtr exportName; +#endif }; inline unsigned NODELETE AbstractModuleRecord::ResolveQuery::Hash::hash(const ResolveQuery& query) @@ -636,7 +640,11 @@ auto AbstractModuleRecord::resolveExportImpl(JSGlobalObject* globalObject, const // Enqueue the tasks in reverse order. for (auto iterator = query.moduleRecord->starExportEntries().rbegin(), end = query.moduleRecord->starExportEntries().rend(); iterator != end; ++iterator) { +#if USE(BUN_JSC_ADDITIONS) + const FiberAwareRefPtr& starModuleName = *iterator; +#else const RefPtr& starModuleName = *iterator; +#endif AbstractModuleRecord* importedModuleRecord = query.moduleRecord->hostResolveImportedModule(globalObject, Identifier::fromUid(vm, starModuleName.get())); RETURN_IF_EXCEPTION(scope, false); pendingTasks.append(Task { ResolveQuery(importedModuleRecord, query.exportName.get()), Type::Query }); @@ -1495,16 +1503,40 @@ unsigned AbstractModuleRecord::innerModuleLinking(JSGlobalObject* globalObject, return index; } +#if USE(BUN_JSC_ADDITIONS) +static String printableName(UniquedStringImpl* uid) +{ + if (isInlinePropertyKey(uid)) { + uintptr_t word = reinterpret_cast(uid); + unsigned len = inlinePropertyKeyLength(word); + const uint8_t* bytes = reinterpret_cast(&word); + if (inlinePropertyKeyIs8Bit(word)) + return WTF::makeString('\'', StringView(std::span { bytes + 1, len }), '\''); + return WTF::makeString('\'', StringView(std::span { reinterpret_cast(bytes + 2), len }), '\''); + } + if (uid->isSymbol()) + return uid; + return WTF::makeString('\'', StringView(uid), '\''); +} +#else static String printableName(const RefPtr& uid) { if (uid->isSymbol()) return uid.get(); return WTF::makeString('\'', StringView(uid.get()), '\''); } +#endif static String printableName(const Identifier& ident) { +#if USE(BUN_JSC_ADDITIONS) + UniquedStringImpl* impl = ident.impl(); + if (isInlinePropertyKey(impl)) + return WTF::makeString('\'', ident.string(), '\''); + return printableName(impl); +#else return printableName(ident.impl()); +#endif } ScriptFetchParameters::Type AbstractModuleRecord::moduleType() const diff --git a/Source/JavaScriptCore/runtime/AbstractModuleRecord.h b/Source/JavaScriptCore/runtime/AbstractModuleRecord.h index dcaac0fb312dc..f4c17387fd8f4 100644 --- a/Source/JavaScriptCore/runtime/AbstractModuleRecord.h +++ b/Source/JavaScriptCore/runtime/AbstractModuleRecord.h @@ -113,9 +113,15 @@ class AbstractModuleRecord : public JSInternalFieldObjectImpl<2> { Identifier localName; }; +#if USE(BUN_JSC_ADDITIONS) + using OrderedIdentifierSet = OrderedHashSet; + using ImportEntries = OrderedHashMap>; + using ExportEntries = OrderedHashMap>; +#else using OrderedIdentifierSet = OrderedHashSet, IdentifierRepHash>; using ImportEntries = OrderedHashMap, ImportEntry, IdentifierRepHash, HashTraits>>; using ExportEntries = OrderedHashMap, ExportEntry, IdentifierRepHash, HashTraits>>; +#endif struct ModuleRequest { Identifier m_specifier; @@ -298,7 +304,11 @@ class AbstractModuleRecord : public JSInternalFieldObjectImpl<2> { // So here, we don't visit each object for GC. The resolution cache map caches the once // looked up correctly resolved resolution, since (1) we rarely looked up the non-resolved one, // and (2) if we cache all the attempts the size of the map becomes infinitely large. +#if USE(BUN_JSC_ADDITIONS) + typedef UncheckedKeyHashMap> Resolutions; +#else typedef UncheckedKeyHashMap, Resolution, IdentifierRepHash, HashTraits>> Resolutions; +#endif Resolutions m_resolutionCache; protected: diff --git a/Source/JavaScriptCore/runtime/BrandedStructure.cpp b/Source/JavaScriptCore/runtime/BrandedStructure.cpp index 7aefcba9a69df..c31631ea6bc1d 100644 --- a/Source/JavaScriptCore/runtime/BrandedStructure.cpp +++ b/Source/JavaScriptCore/runtime/BrandedStructure.cpp @@ -36,6 +36,9 @@ BrandedStructure::BrandedStructure(VM& vm, Structure* previous, UniquedStringImp , m_brand(brandUid) , m_parentBrand(previous->isBrandedStructure() ? previous : nullptr, WriteBarrierEarlyInit) { +#if USE(BUN_JSC_ADDITIONS) + RELEASE_ASSERT(!isInlinePropertyKey(brandUid)); +#endif ASSERT(isBrandedStructure()); } diff --git a/Source/JavaScriptCore/runtime/CacheableIdentifierInlines.h b/Source/JavaScriptCore/runtime/CacheableIdentifierInlines.h index 94814bce0bab6..d65d07ef3240f 100644 --- a/Source/JavaScriptCore/runtime/CacheableIdentifierInlines.h +++ b/Source/JavaScriptCore/runtime/CacheableIdentifierInlines.h @@ -28,6 +28,9 @@ #include "CacheableIdentifier.h" #include "Identifier.h" +#if USE(BUN_JSC_ADDITIONS) +#include "IdentifierInlines.h" +#endif #include "JSCJSValueInlines.h" #include "JSCell.h" #include "VM.h" @@ -60,6 +63,16 @@ inline CacheableIdentifier CacheableIdentifier::createFromSharedStub(UniquedStri inline CacheableIdentifier CacheableIdentifier::createFromCell(JSCell* i) { +#if USE(BUN_JSC_ADDITIONS) + // Phase D: only the small-8-bit-inline form (2..5 Latin-1 chars) is a + // canonical fiber-word key; big-inline / 16-bit-inline span two words and + // must be atomized in place so uid()/getValueImpl() yield a real atom. + if (i->isString()) { + JSString* s = uncheckedDowncast(i); + if (s->isInline() && !(enableIdentifierFiberWords && s->is8Bit() && s->length() <= maxFiberWordKeyLength)) + s->resolveInlineToAtomString(nullptr); + } +#endif return CacheableIdentifier(i); } @@ -90,12 +103,32 @@ inline UniquedStringImpl* CacheableIdentifier::uid() const return &uncheckedDowncast(cell())->uid(); ASSERT(isStringCell()); JSString* string = uncheckedDowncast(cell()); +#if USE(BUN_JSC_ADDITIONS) + // D.4 coherence: return the canonical fiber word for any 2..5-char Latin-1 + // key so PropertyName / PropertyTable::find hit regardless of whether the + // cell is inline or already atom-backed (string literals). + if (enableIdentifierFiberWords && string->isInline() && string->is8Bit() && string->length() <= maxFiberWordKeyLength) + return std::bit_cast(string->inlineFiberWord()); + StringImpl* impl = string->getValueImpl(); + // Fast-bail: a non-inline atom-backed cell with len>5 can never encode as a + // fiber word — return the atom impl directly and skip canonicalFiberWordFor. + if (!string->isInline() && impl->length() > maxFiberWordKeyLength && impl->isAtom()) [[likely]] + return std::bit_cast(impl); + if (uintptr_t fiber = Identifier::canonicalFiberWordFor(impl)) + return std::bit_cast(fiber); + return std::bit_cast(impl); +#else return std::bit_cast(string->getValueImpl()); +#endif } inline bool CacheableIdentifier::isSymbol() const { +#if USE(BUN_JSC_ADDITIONS) + return m_bits && uidIsSymbol(uid()); +#else return m_bits && uid()->isSymbol(); +#endif } inline bool CacheableIdentifier::isPrivateName() const @@ -105,7 +138,11 @@ inline bool CacheableIdentifier::isPrivateName() const inline unsigned CacheableIdentifier::hash() const { +#if USE(BUN_JSC_ADDITIONS) + return uidHash(uid()); +#else return uid()->symbolAwareHash(); +#endif } inline bool CacheableIdentifier::isCacheableIdentifierCell(JSCell* cell) @@ -115,6 +152,12 @@ inline bool CacheableIdentifier::isCacheableIdentifierCell(JSCell* cell) if (!cell->isString()) return false; JSString* string = uncheckedDowncast(cell); +#if USE(BUN_JSC_ADDITIONS) + // Phase D: only small-8-bit-inline (2..5 Latin-1) has a content-unique + // single-word key usable as a uid without atomization. + if (enableIdentifierFiberWords && string->isInline() && string->is8Bit() && string->length() <= maxFiberWordKeyLength) + return true; +#endif if (const StringImpl* impl = string->tryGetValueImpl()) return impl->isAtom(); return false; @@ -134,9 +177,25 @@ inline GCOwnedDataScope CacheableIdentifier::getCachea if (!cell->isString()) return { }; JSString* string = uncheckedDowncast(cell); +#if USE(BUN_JSC_ADDITIONS) + // D.4 coherence: hand back the canonical fiber word so downstream + // PropertyName / CacheableIdentifier::uid() match the parser's key. + if (string->isInline()) { + if (enableIdentifierFiberWords && string->is8Bit() && string->length() <= maxFiberWordKeyLength) + return { cell, std::bit_cast(string->inlineFiberWord()) }; + return { cell, string->resolveInlineToAtomString(nullptr) }; + } + if (const StringImpl* impl = string->tryGetValueImpl(); impl && impl->isAtom()) { + if (uintptr_t fiber = Identifier::canonicalFiberWordFor(impl)) + return { cell, std::bit_cast(fiber) }; + return { cell, static_cast(impl) }; + } + return { }; +#else if (const StringImpl* impl = string->tryGetValueImpl(); impl && impl->isAtom()) return { cell, static_cast(impl) }; return { }; +#endif } inline GCOwnedDataScope CacheableIdentifier::getCacheableIdentifier(JSValue value) @@ -159,10 +218,20 @@ inline bool CacheableIdentifier::isStringCell() const inline void CacheableIdentifier::ensureIsCell(VM& vm) { if (!isCell()) { +#if USE(BUN_JSC_ADDITIONS) + UniquedStringImpl* rep = uid(); + if (uidIsSymbol(rep)) + setCellBits(Symbol::create(vm, static_cast(*rep))); + else if (isInlinePropertyKey(rep)) + setCellBits(JSString::createInlineFromFiber(vm, reinterpret_cast(rep))); + else + setCellBits(jsString(vm, String(static_cast(rep)))); +#else if (uid()->isSymbol()) setCellBits(Symbol::create(vm, static_cast(*uid()))); else setCellBits(jsString(vm, String(static_cast(uid())))); +#endif } ASSERT(isCell()); } diff --git a/Source/JavaScriptCore/runtime/CachedTypes.cpp b/Source/JavaScriptCore/runtime/CachedTypes.cpp index a9afda484de10..6174b909c897b 100644 --- a/Source/JavaScriptCore/runtime/CachedTypes.cpp +++ b/Source/JavaScriptCore/runtime/CachedTypes.cpp @@ -44,6 +44,9 @@ #include "UnlinkedModuleProgramCodeBlock.h" #include "UnlinkedProgramCodeBlock.h" #include "VariableEnvironmentInlines.h" +#if USE(BUN_JSC_ADDITIONS) +#include "InlinePropertyKey.h" +#endif #include #include #include @@ -828,6 +831,61 @@ class CachedUniquedStringImplBase : public VariableLengthObject { class CachedUniquedStringImpl : public CachedUniquedStringImplBase { }; class CachedStringImpl : public CachedUniquedStringImplBase { }; +#if USE(BUN_JSC_ADDITIONS) +// Serializer for FiberAwareRefPtr / FiberAwarePackedRefPtr keys. After the +// Phase-D typedef swap these slots may hold an inline fiber word (bit 1 set) +// instead of a real UniquedStringImpl*. Encode stores the raw word; decode +// reconstitutes the fiber word directly. Real impls fall through to the +// existing CachedUniquedStringImpl path. +template> +class CachedFiberAwareRefPtr : public CachedObject> { + using SourceRefPtr = RefPtr; + +public: + void encode(Encoder& encoder, const UniquedStringImpl* src) + { + if (src && isInlinePropertyKey(src)) { + m_fiberWord = reinterpret_cast(src); + return; + } + m_fiberWord = 0; + m_ptr.encode(encoder, src); + } + + void encode(Encoder& encoder, const SourceRefPtr src) + { + encode(encoder, src.get()); + } + + SourceRefPtr decode(Decoder& decoder) const + { + if (m_fiberWord) + return SourceRefPtr(inlinePropertyKeyAsImpl(m_fiberWord)); + bool isNewAllocation; + UniquedStringImpl* decodedPtr = m_ptr.decode(decoder, isNewAllocation); + if (!decodedPtr) + return nullptr; + if (isNewAllocation) { + decoder.addFinalizer([=] { + WTF::DefaultRefDerefTraits::derefIfNotNull(decodedPtr); + }); + } + auto result = adoptRef(decodedPtr); + result->ref(); + return result; + } + + void decode(Decoder& decoder, SourceRefPtr& src) const + { + src = decode(decoder); + } + +private: + uintptr_t m_fiberWord { 0 }; + CachedPtr m_ptr; +}; +#endif + class CachedString : public VariableLengthObject { public: void encode(Encoder& encoder, const String& string) @@ -1049,7 +1107,11 @@ class CachedCodeBlockRareData : public CachedObject CachedHashMap m_typeProfilerInfoMap; CachedVector m_opProfileControlFlowBytecodeOffsets; CachedVector m_bitVectors; +#if USE(BUN_JSC_ADDITIONS) + CachedVector, IdentifierRepHash>> m_constantIdentifierSets; +#else CachedVector, IdentifierRepHash>> m_constantIdentifierSets; +#endif unsigned m_needsClassFieldInitializer : 1; unsigned m_privateBrandRequirement : 1; }; @@ -1078,7 +1140,11 @@ class CachedExpressionInfo : public CachedObject { CachedArray m_storage; }; +#if USE(BUN_JSC_ADDITIONS) +typedef CachedHashMap, PrivateNameEntry, IdentifierRepHash, HashTraits, PrivateNameEntryHashTraits> CachedPrivateNameEnvironment; +#else typedef CachedHashMap>, PrivateNameEntry, IdentifierRepHash, HashTraits>, PrivateNameEntryHashTraits> CachedPrivateNameEnvironment; +#endif class CachedVariableEnvironmentRareData : public CachedObject { public: @@ -1120,7 +1186,11 @@ class CachedVariableEnvironment : public CachedObject { private: bool m_isEverythingCaptured; bool m_hasAwaitUsingDeclaration; +#if USE(BUN_JSC_ADDITIONS) + CachedInlineMap, VariableEnvironmentEntry, VariableEnvironment::inlineMapCapacity, IdentifierRepHash, HashTraits, VariableEnvironmentEntryHashTraits> m_map; +#else CachedInlineMap>, VariableEnvironmentEntry, VariableEnvironment::inlineMapCapacity, IdentifierRepHash, HashTraits>, VariableEnvironmentEntryHashTraits> m_map; +#endif CachedPtr m_rareData; }; @@ -1158,7 +1228,11 @@ class CachedCompactTDZEnvironment : public CachedObject { } private: +#if USE(BUN_JSC_ADDITIONS) + CachedVector> m_variables; +#else CachedVector>> m_variables; +#endif unsigned m_hash; }; @@ -1286,7 +1360,11 @@ class CachedSymbolTable : public CachedObject { } private: +#if USE(BUN_JSC_ADDITIONS) + CachedHashMap, CachedSymbolTableEntry, IdentifierRepHash, HashTraits, SymbolTableIndexHashTraits> m_map; +#else CachedHashMap, CachedSymbolTableEntry, IdentifierRepHash, HashTraits>, SymbolTableIndexHashTraits> m_map; +#endif ScopeOffset m_maxScopeOffset; unsigned m_usesSloppyEval : 1; unsigned m_nestedLexicalScope : 1; diff --git a/Source/JavaScriptCore/runtime/CodeCache.h b/Source/JavaScriptCore/runtime/CodeCache.h index de2bea81cc155..a7bbce22ac68b 100644 --- a/Source/JavaScriptCore/runtime/CodeCache.h +++ b/Source/JavaScriptCore/runtime/CodeCache.h @@ -54,7 +54,11 @@ class ProgramExecutable; class SourceCode; class VM; +#if USE(BUN_JSC_ADDITIONS) +// TDZEnvironment is defined in VariableEnvironment.h with FiberAwareRefPtr; avoid a conflicting redefinition here. +#else using TDZEnvironment = UncheckedKeyHashSet, IdentifierRepHash>; +#endif namespace CodeCacheInternal { static constexpr bool verbose = false; diff --git a/Source/JavaScriptCore/runtime/Error.cpp b/Source/JavaScriptCore/runtime/Error.cpp index b5514f770a289..10f04dcdba967 100644 --- a/Source/JavaScriptCore/runtime/Error.cpp +++ b/Source/JavaScriptCore/runtime/Error.cpp @@ -354,12 +354,12 @@ Exception* throwSyntaxError(JSGlobalObject* globalObject, ThrowScope& scope, con JSValue throwDOMAttributeGetterTypeError(JSGlobalObject* globalObject, ThrowScope& scope, const ClassInfo* classInfo, PropertyName propertyName) { - return throwException(globalObject, scope, createGetterTypeError(globalObject, makeDOMAttributeGetterTypeErrorMessage(classInfo->className, String(propertyName.uid())))); + return throwException(globalObject, scope, createGetterTypeError(globalObject, makeDOMAttributeGetterTypeErrorMessage(classInfo->className, String(propertyName.publicName())))); } JSValue throwDOMAttributeSetterTypeError(JSGlobalObject* globalObject, ThrowScope& scope, const ClassInfo* classInfo, PropertyName propertyName) { - return throwException(globalObject, scope, createTypeError(globalObject, makeDOMAttributeSetterTypeErrorMessage(classInfo->className, String(propertyName.uid())))); + return throwException(globalObject, scope, createTypeError(globalObject, makeDOMAttributeSetterTypeErrorMessage(classInfo->className, String(propertyName.publicName())))); } JSObject* createError(JSGlobalObject* globalObject, const String& message) diff --git a/Source/JavaScriptCore/runtime/GetPutInfo.h b/Source/JavaScriptCore/runtime/GetPutInfo.h index e9cd7111c3462..24c9ec77ec4aa 100644 --- a/Source/JavaScriptCore/runtime/GetPutInfo.h +++ b/Source/JavaScriptCore/runtime/GetPutInfo.h @@ -26,6 +26,9 @@ #pragma once #include "ECMAMode.h" +#if USE(BUN_JSC_ADDITIONS) +#include "InlinePropertyKey.h" +#endif #include #include @@ -217,7 +220,11 @@ struct ResolveOp { JSLexicalEnvironment* lexicalEnvironment; WatchpointSet* watchpointSet; uintptr_t operand; +#if USE(BUN_JSC_ADDITIONS) + FiberAwareRefPtr importedName; +#else RefPtr importedName; +#endif }; class GetPutInfo { diff --git a/Source/JavaScriptCore/runtime/HasOwnPropertyCache.h b/Source/JavaScriptCore/runtime/HasOwnPropertyCache.h index 490d79483a7d7..da3ecec4005d9 100644 --- a/Source/JavaScriptCore/runtime/HasOwnPropertyCache.h +++ b/Source/JavaScriptCore/runtime/HasOwnPropertyCache.h @@ -30,6 +30,10 @@ #include "Structure.h" #include +#if USE(BUN_JSC_ADDITIONS) +#include "InlinePropertyKey.h" +#endif + WTF_ALLOW_UNSAFE_BUFFER_USAGE_BEGIN namespace JSC { @@ -47,7 +51,11 @@ class alignas(8) HasOwnPropertyCache { static constexpr ptrdiff_t offsetOfImpl() { return OBJECT_OFFSETOF(Entry, impl); } static constexpr ptrdiff_t offsetOfResult() { return OBJECT_OFFSETOF(Entry, result); } +#if USE(BUN_JSC_ADDITIONS) + FiberAwareRefPtr impl; +#else RefPtr impl; +#endif StructureID structureID; bool result { false }; }; @@ -70,7 +78,11 @@ class alignas(8) HasOwnPropertyCache { ALWAYS_INLINE static uint32_t hash(StructureID structureID, UniquedStringImpl* impl) { +#if USE(BUN_JSC_ADDITIONS) + return std::bit_cast(structureID) + uidHash(impl); +#else return std::bit_cast(structureID) + impl->hash(); +#endif } ALWAYS_INLINE std::optional get(Structure* structure, PropertyName propName) @@ -110,7 +122,11 @@ class alignas(8) HasOwnPropertyCache { UniquedStringImpl* impl = propName.uid(); StructureID id = structure->id(); uint32_t index = HasOwnPropertyCache::hash(id, impl) & mask; +#if USE(BUN_JSC_ADDITIONS) + std::bit_cast(this)[index] = Entry { FiberAwareRefPtr(impl), id, result }; +#else std::bit_cast(this)[index] = Entry { RefPtr(impl), id, result }; +#endif } } diff --git a/Source/JavaScriptCore/runtime/HashMapHelper.h b/Source/JavaScriptCore/runtime/HashMapHelper.h index 7c007167b20d5..0eec16a090d82 100644 --- a/Source/JavaScriptCore/runtime/HashMapHelper.h +++ b/Source/JavaScriptCore/runtime/HashMapHelper.h @@ -90,6 +90,15 @@ ALWAYS_INLINE uint32_t jsMapHashImpl(JSGlobalObject* globalObject, VM& vm, JSVal if (value.isString()) { auto scope = DECLARE_THROW_SCOPE(vm); +#if USE(BUN_JSC_ADDITIONS) + // Hash inline bytes directly; resolving would allocate a StringImpl. + if (JSString* str = asString(value); str->isInline()) { + unsigned len = str->length(); + if (str->is8Bit()) + return StringHasher::computeHashAndMaskTop8Bits(std::span { str->inlineData8(), len }); + return StringHasher::computeHashAndMaskTop8Bits(std::span { str->inlineData16(), len }); + } +#endif auto wtfString = asString(value)->value(globalObject); if constexpr (expection == ExceptionExpectation::CanThrow) RETURN_IF_EXCEPTION(scope, UINT_MAX); diff --git a/Source/JavaScriptCore/runtime/Identifier.cpp b/Source/JavaScriptCore/runtime/Identifier.cpp index cbc5f68840475..98acb8be9328f 100644 --- a/Source/JavaScriptCore/runtime/Identifier.cpp +++ b/Source/JavaScriptCore/runtime/Identifier.cpp @@ -57,6 +57,23 @@ Identifier Identifier::from(VM& vm, double value) void Identifier::dump(PrintStream& out) const { +#if USE(BUN_JSC_ADDITIONS) + if (impl()) { + if (isInlinePropertyKey(impl())) { + // Avoid string(): dump() runs on JIT compiler threads and must not + // materialize the fiber word via the thread-local atom table. + out.print(stringWithoutAtomizing()); + return; + } + if (uidIsSymbol(impl())) { + auto* symbol = static_cast(impl()); + if (symbol->isPrivate()) + out.print("PrivateSymbol."); + } + out.print(impl()); + } else + out.print(""); +#else if (impl()) { if (impl()->isSymbol()) { auto* symbol = static_cast(impl()); @@ -66,6 +83,7 @@ void Identifier::dump(PrintStream& out) const out.print(impl()); } else out.print(""); +#endif } #ifndef NDEBUG diff --git a/Source/JavaScriptCore/runtime/Identifier.h b/Source/JavaScriptCore/runtime/Identifier.h index c2df39e6c3254..2f27cd1d37a74 100644 --- a/Source/JavaScriptCore/runtime/Identifier.h +++ b/Source/JavaScriptCore/runtime/Identifier.h @@ -28,6 +28,10 @@ #include #include +#if USE(BUN_JSC_ADDITIONS) +#include "InlinePropertyKey.h" +#endif + namespace JSC { class CallFrame; @@ -84,6 +88,210 @@ ALWAYS_INLINE std::optional parseIndex(const StringImpl& impl) return impl.is8Bit() ? parseIndex(impl.span8()) : parseIndex(impl.span16()); } +#if USE(BUN_JSC_ADDITIONS) +class Identifier { + friend class Structure; +public: + Identifier() = default; + enum class EmptyIdentifierFlag { EmptyIdentifier }; + Identifier(EmptyIdentifierFlag) + { + auto* empty = StringImpl::empty(); + empty->ref(); + m_bits = reinterpret_cast(empty); + ASSERT(empty->isAtom()); + } + + // Single-word storage (sizeof(Identifier) == sizeof(void*)): copy/move/ + // assign/dtor touch only m_bits plus a fiber-aware ref/deref. string() is + // by-value so no per-instance materialization cache is needed. + Identifier(const Identifier& other) + : m_bits(other.m_bits) + { + if (m_bits) + uidRef(reinterpret_cast(m_bits)); + } + + Identifier(Identifier&& other) + : m_bits(std::exchange(other.m_bits, 0)) + { + } + + ~Identifier() + { + if (m_bits) + uidDeref(reinterpret_cast(m_bits)); + } + + Identifier& operator=(const Identifier& other) + { + uintptr_t newBits = other.m_bits; + if (newBits) + uidRef(reinterpret_cast(newBits)); + uintptr_t oldBits = std::exchange(m_bits, newBits); + if (oldBits) + uidDeref(reinterpret_cast(oldBits)); + return *this; + } + + Identifier& operator=(Identifier&& other) + { + uintptr_t oldBits = std::exchange(m_bits, std::exchange(other.m_bits, 0)); + if (oldBits) + uidDeref(reinterpret_cast(oldBits)); + return *this; + } + + enum class FromFiberWordTag { T }; + static Identifier fromFiberWord(uintptr_t word) { return Identifier(FromFiberWordTag::T, word); } + + AtomString string() const + { + if (!m_bits) + return nullAtom(); + if (isInlinePropertyKey(m_bits)) [[unlikely]] { + unsigned len = inlinePropertyKeyLength(m_bits); + const uint8_t* bytes = reinterpret_cast(&m_bits); + if (inlinePropertyKeyIs8Bit(m_bits)) + return AtomString(std::span { bytes + 1, len }); + return AtomString(std::span { reinterpret_cast(bytes + 2), len }); + } + return AtomString(reinterpret_cast(m_bits)); + } + + // May return a fiber-word-tagged pointer; callers are phase-A shimmed. + UniquedStringImpl* impl() const { return reinterpret_cast(m_bits); } + + RefPtr releaseImpl() + { + AtomString result = string(); + uintptr_t oldBits = std::exchange(m_bits, 0); + if (oldBits) + uidDeref(reinterpret_cast(oldBits)); + return result.releaseImpl(); + } + + int length() const { return m_bits ? static_cast(uidLength(reinterpret_cast(m_bits))) : 0; } + + // Bypass string() so a fiber-word decode never touches the thread-local + // AtomStringTable: FTL compiler threads reach here via + // CodeBlock::inferredName() -> ecmaName().utf8(), and an atom materialized + // there would later be destroyed on the main thread's GC sweep. + CString ascii() const { return stringWithoutAtomizing().ascii(); } + CString utf8() const { return stringWithoutAtomizing().utf8(); } + + String stringWithoutAtomizing() const + { + if (!m_bits) + return String(); + if (isInlinePropertyKey(m_bits)) { + unsigned len = inlinePropertyKeyLength(m_bits); + const uint8_t* bytes = reinterpret_cast(&m_bits); + if (inlinePropertyKeyIs8Bit(m_bits)) + return String(std::span { bytes + 1, len }); + return String(std::span { reinterpret_cast(bytes + 2), len }); + } + return String(reinterpret_cast(m_bits)); + } + + // There's 2 functions to construct Identifier from string, (1) fromString and (2) fromUid. + // They have different meanings in keeping or discarding symbol-ness of strings. + // (1): fromString + // Just construct Identifier from string. String held by Identifier is always atomized. + // Symbol-ness of StringImpl*, which represents that the string is inteded to be used for ES6 Symbols, is discarded. + // So a constructed Identifier never represents a symbol. + // (2): fromUid + // `StringImpl* uid` represents ether String or Symbol property. + // fromUid keeps symbol-ness of provided StringImpl* while fromString discards it. + // Use fromUid when constructing Identifier from StringImpl* which may represent symbols. + + static Identifier fromString(VM&, ASCIILiteral); + static Identifier fromString(VM&, std::span); + static Identifier fromString(VM&, std::span); + static Identifier fromString(VM&, const String&); + static Identifier fromString(VM&, AtomStringImpl*); + static Identifier fromString(VM&, Ref&&); + static Identifier fromString(VM&, const AtomString&); + static Identifier fromString(VM&, SymbolImpl*); + + static Identifier NODELETE fromUid(VM&, UniquedStringImpl* uid); + static Identifier fromUid(const PrivateName&); + static Identifier fromUid(SymbolImpl&); + + static inline Identifier createLatin1(VM& vm, std::span string); // Defined in IdentifierInlines.h + + JS_EXPORT_PRIVATE static Identifier from(VM&, unsigned y); + JS_EXPORT_PRIVATE static Identifier from(VM&, int y); + JS_EXPORT_PRIVATE static Identifier from(VM&, double y); + ALWAYS_INLINE static Identifier from(VM& vm, uint64_t y) + { + if (static_cast(y) == y) + return from(vm, static_cast(y)); + ASSERT(static_cast(static_cast(y)) == y); + return from(vm, static_cast(y)); + } + + bool isNull() const { return !m_bits; } + // Matches AtomString::isEmpty(): null OR zero-length. BreakNode/ContinueNode + // pass a null-m_bits Identifier for the unlabeled case and test isEmpty(). + bool isEmpty() const { return !m_bits || (!isInlinePropertyKey(m_bits) && !reinterpret_cast(m_bits)->length()); } + bool isSymbol() const { return m_bits && uidIsSymbol(reinterpret_cast(m_bits)); } + bool isPrivateName() const { return isSymbol() && static_cast(impl())->isPrivate(); } + + friend bool operator==(const Identifier&, const Identifier&); + + static bool equal(const StringImpl*, std::span); + static bool equal(const StringImpl*, std::span); + static bool equal(const StringImpl* a, const StringImpl* b) { return ::equal(a, b); } + + void dump(PrintStream&) const; + + // Phase D.4 coherence: every construction path for a 2..5-char Latin-1 + // name must yield the same encodeInline8 fiber word so PropertyTable's + // pointer-identity compare hits regardless of which producer ran. + static uintptr_t canonicalFiberWordFor(const StringImpl*); // Defined in IdentifierInlines.h + +private: + uintptr_t m_bits { 0 }; + + Identifier(FromFiberWordTag, uintptr_t word) + : m_bits(word) + { + ASSERT(isInlinePropertyKey(word)); + } + + inline Identifier(VM&, std::span); // Defined in IdentifierInlines.h + inline Identifier(VM&, std::span); // Defined in IdentifierInlines.h + ALWAYS_INLINE Identifier(VM&, ASCIILiteral); // Defined in IdentifierInlines.h + inline Identifier(VM&, AtomStringImpl*); // Defined in IdentifierInlines.h + inline Identifier(VM&, const AtomString&); // Defined in IdentifierInlines.h + inline Identifier(VM&, const String&); + inline Identifier(VM&, StringImpl*); + inline Identifier(VM&, Ref&&); // Defined in IdentifierInlines.h + + Identifier(SymbolImpl& uid) + { + uid.ref(); + m_bits = reinterpret_cast(&uid); + } + + static bool equal(const Identifier& a, const Identifier& b) { return a.m_bits == b.m_bits; } + + template inline static Ref add(VM&, std::span); // Defined in IdentifierInlines.h + static Ref add8(VM&, std::span); + template ALWAYS_INLINE static constexpr bool canUseSingleCharacterString(T); + + static Ref add(VM&, StringImpl*); + inline static Ref add(VM&, ASCIILiteral); // Defined in IdentifierInlines.h + +#ifndef NDEBUG + JS_EXPORT_PRIVATE static void checkCurrentAtomStringTable(VM&); +#else + JS_EXPORT_PRIVATE NO_RETURN_DUE_TO_CRASH static void checkCurrentAtomStringTable(VM&); +#endif +}; +static_assert(sizeof(Identifier) == sizeof(void*), "Identifier must stay single-word so parser/bytecompiler copies are one ref/deref"); +#else class Identifier { friend class Structure; public: @@ -185,6 +393,7 @@ class Identifier { JS_EXPORT_PRIVATE NO_RETURN_DUE_TO_CRASH static void checkCurrentAtomStringTable(VM&); #endif }; +#endif template <> ALWAYS_INLINE constexpr bool Identifier::canUseSingleCharacterString(Latin1Character) { @@ -204,22 +413,60 @@ inline bool operator==(const Identifier& a, const Identifier& b) inline bool Identifier::equal(const StringImpl* r, std::span s) { +#if USE(BUN_JSC_ADDITIONS) + if (isInlinePropertyKey(reinterpret_cast(r))) [[unlikely]] { + uintptr_t w = reinterpret_cast(r); + if (!inlinePropertyKeyIs8Bit(w)) + return false; + auto span = inlinePropertyKeySpan8(w); + return span.size() == s.size() && !memcmp(span.data(), s.data(), s.size()); + } +#endif return WTF::equal(r, s); } inline bool Identifier::equal(const StringImpl* r, std::span s) { +#if USE(BUN_JSC_ADDITIONS) + if (isInlinePropertyKey(reinterpret_cast(r))) [[unlikely]] { + uintptr_t w = reinterpret_cast(r); + if (!inlinePropertyKeyIs8Bit(w)) + return false; + auto span = inlinePropertyKeySpan8(w); + if (span.size() != s.size()) + return false; + for (size_t i = 0; i < s.size(); ++i) + if (span[i] != s[i]) + return false; + return true; + } +#endif return WTF::equal(r, s); } ALWAYS_INLINE std::optional parseIndex(const Identifier& identifier) { +#if USE(BUN_JSC_ADDITIONS) + auto uid = identifier.impl(); + if (!uid || uidIsSymbol(uid)) + return std::nullopt; + if (isInlinePropertyKey(uid)) { + // D.4 coherence means numeric strings "42".."9999999" arrive here as + // fiber words; decode so obj["42"] still hits the indexed-property path. + uintptr_t w = reinterpret_cast(uid); + if (inlinePropertyKeyIs8Bit(w)) + return parseIndex(inlinePropertyKeySpan8(w)); + return std::nullopt; + } + return parseIndex(*uid); +#else auto uid = identifier.impl(); if (!uid) return std::nullopt; if (uid->isSymbol()) return std::nullopt; return parseIndex(*uid); +#endif } JSValue identifierToJSValue(VM&, const Identifier&); @@ -232,7 +479,14 @@ JSValue identifierToSafePublicJSValue(VM&, const Identifier&); // crashes in code that somehow dangled a StringImpl. // https://bugs.webkit.org/show_bug.cgi?id=150137 struct IdentifierRepHash : PtrHash> { - static unsigned hash(const UniquedStringImpl* key) { return key->existingSymbolAwareHash(); } + static unsigned hash(const UniquedStringImpl* key) + { +#if USE(BUN_JSC_ADDITIONS) + return uidHash(key); +#else + return key->existingSymbolAwareHash(); +#endif + } static constexpr bool hasHashInValue = true; }; @@ -241,14 +495,22 @@ struct IdentifierMapIndexHashTraits : HashTraits { static constexpr bool emptyValueIsZero = false; }; +#if USE(BUN_JSC_ADDITIONS) +typedef UncheckedKeyHashSet IdentifierSet; +typedef UncheckedKeyHashMap, IdentifierMapIndexHashTraits> IdentifierMap; +typedef UncheckedKeyHashMap, IdentifierMapIndexHashTraits> BorrowedIdentifierMap; +#else typedef UncheckedKeyHashSet, IdentifierRepHash> IdentifierSet; typedef UncheckedKeyHashMap, int, IdentifierRepHash, HashTraits>, IdentifierMapIndexHashTraits> IdentifierMap; typedef UncheckedKeyHashMap, IdentifierMapIndexHashTraits> BorrowedIdentifierMap; +#endif } // namespace JSC namespace WTF { +// Identifier is a single uintptr_t (m_bits); equality is m_bits == m_bits, so +// memcmp comparison is sound again now that the materialized-string cache is gone. template <> struct VectorTraits : SimpleClassVectorTraits { }; } // namespace WTF diff --git a/Source/JavaScriptCore/runtime/IdentifierInlines.h b/Source/JavaScriptCore/runtime/IdentifierInlines.h index 54ee571f61b0b..0f9d82b683169 100644 --- a/Source/JavaScriptCore/runtime/IdentifierInlines.h +++ b/Source/JavaScriptCore/runtime/IdentifierInlines.h @@ -33,12 +33,166 @@ namespace JSC { +#if USE(BUN_JSC_ADDITIONS) +// Phase D.4 canonical-representation coherence: a 2..7-char Latin-1 name has +// exactly one m_bits value — the encodeInline8 fiber word. Every Identifier +// construction path funnels through this so PropertyTable's key==entry.key() +// pointer compare hits whether the key came from the parser span ctor, +// JSString::toIdentifier, or any fromString()/fromUid() caller. +[[gnu::hot]] ALWAYS_INLINE uintptr_t Identifier::canonicalFiberWordFor(const StringImpl* impl) +{ + if constexpr (!enableIdentifierFiberWords) + return 0; + if (!impl) + return 0; + // Length bail first: real-world identifiers are overwhelmingly >5 chars, so + // take the cheap m_length load before touching m_hashAndFlags for isSymbol(). + unsigned len = impl->length(); + if (len < 2 || len > maxFiberWordKeyLength) [[likely]] + return 0; + if (impl->isSymbol()) [[unlikely]] + return 0; + if (impl->is8Bit()) [[likely]] + return JSString::encodeInline8(impl->span8()); + auto span16 = impl->span16(); + Latin1Character narrowed[JSString::maxInlineLength8]; + for (unsigned i = 0; i < len; ++i) { + char16_t c = span16[i]; + if (c > 0xff) + return 0; + narrowed[i] = static_cast(c); + } + return JSString::encodeInline8({ narrowed, len }); +} + +inline Identifier::Identifier(VM& vm, std::span string) +{ + // Phase D.2 producer: short Latin-1 identifiers live as a tagged fiber word + // (no AtomStringImpl materialized). size 0/1 and >5 fall through to add(). + if constexpr (enableIdentifierFiberWords) { + if (string.size() >= 2 && string.size() <= maxFiberWordKeyLength) { + m_bits = JSString::encodeInline8(string); + return; + } + } + m_bits = reinterpret_cast(&add(vm, string).leakRef()); +} +#else inline Identifier::Identifier(VM& vm, std::span string) : m_string(add(vm, string)) { ASSERT(m_string.impl()->isAtom()); } +#endif +#if USE(BUN_JSC_ADDITIONS) +inline Identifier::Identifier(VM& vm, std::span string) +{ + // D.4 coherence: a 16-bit span whose content fits Latin-1 must yield the + // same fiber word the Latin-1 span ctor would. + if constexpr (enableIdentifierFiberWords) { + if (string.size() >= 2 && string.size() <= maxFiberWordKeyLength) { + Latin1Character narrowed[JSString::maxInlineLength8]; + bool allLatin1 = true; + for (size_t i = 0; i < string.size(); ++i) { + char16_t c = string[i]; + if (c > 0xff) { allLatin1 = false; break; } + narrowed[i] = static_cast(c); + } + if (allLatin1) { + m_bits = JSString::encodeInline8({ narrowed, string.size() }); + return; + } + } + } + m_bits = reinterpret_cast(&add(vm, string).leakRef()); + ASSERT(reinterpret_cast(m_bits)->isAtom()); +} + +ALWAYS_INLINE Identifier::Identifier(VM& vm, ASCIILiteral literal) +{ + // D.4 coherence: ASCII is Latin-1; 2..5 chars must be a fiber word. + if constexpr (enableIdentifierFiberWords) { + size_t len = literal.length(); + if (len >= 2 && len <= maxFiberWordKeyLength) { + m_bits = JSString::encodeInline8(literal.span8()); + return; + } + } + m_bits = reinterpret_cast(&add(vm, literal).leakRef()); + ASSERT(reinterpret_cast(m_bits)->isAtom()); +} + +inline Identifier::Identifier(VM& vm, AtomStringImpl* string) +{ + if (string) { + if (uintptr_t fiber = canonicalFiberWordFor(string)) { + m_bits = fiber; + } else { + string->ref(); + m_bits = reinterpret_cast(string); + } + } +#ifndef NDEBUG + checkCurrentAtomStringTable(vm); + if (string) + ASSERT_WITH_MESSAGE(!string->length() || string->isSymbol() || AtomStringImpl::isInAtomStringTable(string), "The atomic string comes from an other thread!"); +#else + UNUSED_PARAM(vm); +#endif +} + +inline Identifier::Identifier(VM& vm, const AtomString& string) +{ + if (auto* rep = string.impl()) { + if (uintptr_t fiber = canonicalFiberWordFor(rep)) { + m_bits = fiber; + } else { + rep->ref(); + m_bits = reinterpret_cast(static_cast(rep)); + } + } +#ifndef NDEBUG + checkCurrentAtomStringTable(vm); + if (!string.isNull()) + ASSERT_WITH_MESSAGE(!string.length() || string.impl()->isSymbol() || AtomStringImpl::isInAtomStringTable(string.impl()), "The atomic string comes from an other thread!"); +#else + UNUSED_PARAM(vm); +#endif +} + +inline Identifier::Identifier(VM& vm, const String& string) +{ + if (uintptr_t fiber = canonicalFiberWordFor(string.impl())) { + m_bits = fiber; + return; + } + m_bits = reinterpret_cast(&add(vm, string.impl()).leakRef()); + ASSERT(reinterpret_cast(m_bits)->isAtom()); +} + +inline Identifier::Identifier(VM& vm, StringImpl* rep) +{ + if (uintptr_t fiber = canonicalFiberWordFor(rep)) { + m_bits = fiber; + return; + } + m_bits = reinterpret_cast(&add(vm, rep).leakRef()); + ASSERT(reinterpret_cast(m_bits)->isAtom()); +} + +inline Identifier::Identifier(VM&, Ref&& impl) +{ + // D.4 coherence: JSString::toIdentifier / JSRopeString::toIdentifier land + // here for non-inline strings — must match the parser's fiber word for + // the same 2..7-char Latin-1 content. + if (uintptr_t fiber = canonicalFiberWordFor(impl.ptr())) { + m_bits = fiber; + return; + } + m_bits = reinterpret_cast(&impl.leakRef()); +} +#else inline Identifier::Identifier(VM& vm, std::span string) : m_string(add(vm, string)) { @@ -86,6 +240,7 @@ inline Identifier::Identifier(VM& vm, StringImpl* rep) { ASSERT(m_string.impl()->isAtom()); } +#endif inline Ref Identifier::add(VM& vm, ASCIILiteral literal) { @@ -106,6 +261,21 @@ Ref Identifier::add(VM& vm, std::span string) if (string.empty()) return *static_cast(StringImpl::empty()); +#if USE(BUN_JSC_ADDITIONS) + // Short Latin-1: the inline fiber word is a content-unique key. One + // direct-mapped compare replaces AtomStringImpl::add's hash+probe. + if constexpr (std::is_same_v) { + if (string.size() <= JSString::maxInlineLength8) { + uintptr_t fiber = JSString::encodeInline8(string); + if (auto* cached = vm.inlineAtomCache.lookup(fiber)) + return *cached; + auto atom = AtomStringImpl::add(string); + vm.inlineAtomCache.insert(fiber, atom.get()); + return atom.releaseNonNull(); + } + } +#endif + return *AtomStringImpl::add(string); } @@ -124,9 +294,37 @@ inline Identifier Identifier::createLatin1(VM& vm, std::span str SUPPRESS_NODELETE inline Identifier Identifier::fromUid(VM& vm, UniquedStringImpl* uid) { +#if USE(BUN_JSC_ADDITIONS) + // D.4 coherence: uid may be a fiber word (from Identifier::impl() or + // CacheableIdentifier::uid()). Only the encodeInline8 form (8-bit, 2..7 + // chars) is canonical for m_bits — encodeInline16 / big-inline words must + // be decoded and re-routed so PropertyTable pointer compares still hit. + if (isInlinePropertyKey(uid)) { + uintptr_t w = reinterpret_cast(uid); + unsigned len = inlinePropertyKeyLength(w); + if (inlinePropertyKeyIs8Bit(w) && len >= 2 && len <= maxFiberWordKeyLength) + return fromFiberWord(w); + // Non-canonical inline. A big-inline word is detached from its cell + // and payload exceeds the 8-byte word — upstream producer bug; ASSERT + // and cap the read to stay in-bounds. + const uint8_t* bytes = reinterpret_cast(&w); + if (inlinePropertyKeyIs8Bit(w)) { + ASSERT_WITH_MESSAGE(len <= JSString::maxInlineLength8, "big-inline fiber word detached from cell"); + unsigned take = len <= JSString::maxInlineLength8 ? len : JSString::maxInlineLength8; + return Identifier(vm, std::span { bytes + 1, take }); + } + ASSERT_WITH_MESSAGE(len <= JSString::maxInlineLength16, "big-inline fiber word detached from cell"); + unsigned take = len <= JSString::maxInlineLength16 ? len : JSString::maxInlineLength16; + return Identifier(vm, std::span { reinterpret_cast(bytes + 2), take }); + } + if (!uid || !uid->isSymbol()) + return Identifier(vm, static_cast(uid)); + return static_cast(*uid); +#else if (!uid || !uid->isSymbol()) return Identifier(vm, uid); return static_cast(*uid); +#endif } inline Identifier Identifier::fromUid(const PrivateName& name) diff --git a/Source/JavaScriptCore/runtime/InlinePropertyKey.h b/Source/JavaScriptCore/runtime/InlinePropertyKey.h new file mode 100644 index 0000000000000..a743538f61073 --- /dev/null +++ b/Source/JavaScriptCore/runtime/InlinePropertyKey.h @@ -0,0 +1,195 @@ +/* + * Copyright (C) 2024 Apple Inc. All rights reserved. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + */ + +#pragma once + +#if USE(BUN_JSC_ADDITIONS) + +#include +#include +#include +#include +#include +#include +#include + +namespace JSC { + +// A UniquedStringImpl* slot may instead hold an inline-encoded fiber word +// (bit 1 set, bit 0 clear; see JSString::isInlineInPointer). Real impls are +// 8-byte aligned so the low 3 bits distinguish. The word is content-unique for +// 2..7 Latin-1 / 1..3 UTF-16 characters, so pointer-identity comparisons and +// PtrHash-style hashing remain sound without dereferencing. +// +// These helpers shim the handful of call sites that dereference a uid for +// isSymbol() / hash() / length() / ref() / deref() so they branch on the tag +// first. Call sites that only compare or store the raw value need no change. + +static constexpr uintptr_t inlinePropertyKeyTag = 0x2; +static constexpr uintptr_t inlinePropertyKeyTagMask = 0x3; + +// Canonical fiber-word keys cap at 5 chars so the whole word fits in bits 0..47. +// CompactPropertyTableEntry / StructureTransitionTable::Hash::Key pack attributes +// into bits 48..63 of a UniquedStringImpl* slot; a 6/7-char payload there would +// be truncated. 6..7-char names stay atom-backed for D.4 coherence. +static constexpr unsigned maxFiberWordKeyLength = 5; + +// Compile-time killswitch for the D.2/D.4 producer paths. Flip to false to +// disable fiber-word Identifiers in one place (debug: 'shims broken' vs +// 'producer reached unguarded site') without reverting commits. +// NOT a perf escape hatch on its own: with this =false and no other change, +// JetStream2 Total Score was 188.004 vs baseline median 200.169 (−6.1%, +// /tmp/killswitch-survey2.log) — the phase-A shim overhead persists with zero +// fiber words produced. synthesize-measure-commit therefore CANNOT fall back to +// this flag alone to reach baseline+2%; the targeted perf fixes must stay. +static constexpr bool enableIdentifierFiberWords = true; + +ALWAYS_INLINE bool isInlinePropertyKey(const UniquedStringImpl* impl) +{ + // Real impls are 8-byte aligned (low 3 bits == 0) and fiber words set bit 1, + // so a single bit-test on the tag bit suffices — cheaper than mask-then-compare. + return reinterpret_cast(impl) & inlinePropertyKeyTag; +} + +ALWAYS_INLINE bool isInlinePropertyKey(uintptr_t word) +{ + return word & inlinePropertyKeyTag; +} + +ALWAYS_INLINE uintptr_t inlinePropertyKeyWord(const UniquedStringImpl* impl) +{ + return reinterpret_cast(impl); +} + +ALWAYS_INLINE UniquedStringImpl* inlinePropertyKeyAsImpl(uintptr_t word) +{ + return reinterpret_cast(word); +} + +// Layout matches JSString's inline encoding: bit 2 is is8Bit, bits 3..6 are +// length, payload at byte 1 (8-bit) or byte 2 (16-bit). +ALWAYS_INLINE bool inlinePropertyKeyIs8Bit(uintptr_t word) { return word & 0x4; } +ALWAYS_INLINE unsigned inlinePropertyKeyLength(uintptr_t word) { return static_cast(word >> 3) & 0xf; } + +// Span into the caller's fiber-word storage (taken by reference so the returned +// span stays valid for the enclosing full-expression / member lifetime). +ALWAYS_INLINE std::span inlinePropertyKeySpan8(const uintptr_t& word) +{ + ASSERT(isInlinePropertyKey(word)); + ASSERT(inlinePropertyKeyIs8Bit(word)); + const uint8_t* bytes = reinterpret_cast(&word); + return std::span { bytes + 1, inlinePropertyKeyLength(word) }; +} + +ALWAYS_INLINE unsigned inlinePropertyKeyHash(uintptr_t word) +{ + // Must match StringImpl::hash() for equal content so a fiber-word key + // compares against an AtomStringImpl* key with the same hash. + unsigned len = inlinePropertyKeyLength(word); + const uint8_t* bytes = reinterpret_cast(&word); + if (inlinePropertyKeyIs8Bit(word)) + return StringHasher::computeHashAndMaskTop8Bits(std::span { bytes + 1, len }); + return StringHasher::computeHashAndMaskTop8Bits(std::span { reinterpret_cast(bytes + 2), len }); +} + +// Shims: safe to call on either a real impl or a fiber word. + +ALWAYS_INLINE bool uidIsSymbol(const UniquedStringImpl* impl) +{ + uintptr_t bits = reinterpret_cast(impl); + if (bits & inlinePropertyKeyTag) + return false; + return impl->isSymbol(); +} + +// 64-bit Fibonacci / golden-ratio multiplicative hash constant (2^64 / phi, odd). +// Kept here so the C++ uidHash() and its DFG/FTL/AssemblyHelpers JIT mirrors all +// reference the same value. +static constexpr uint64_t uidHashMultiplier = 0x9E3779B97F4A7C15ULL; + +ALWAYS_INLINE unsigned uidHash(const UniquedStringImpl* impl) +{ + // Option A — branch-free Fibonacci mix of the raw pointer word. D.4 single-rep + // guarantees one m_bits per content, so hashing the word is sound for both real + // impls and fiber words. Picked over Option B (tag-branch → real impl->hash()): + // B adds a cond+uncond jump to every JIT MegamorphicCache/HasOwnProperty probe, + // and /tmp/fix3-pertest3.log (post-8f5292b1d0 stack) showed Babylon −9.56% with B + // in place; /tmp/killswitch-result.txt records A as a required shim-cost fix. + // Not gated on enableIdentifierFiberWords: A is correct for both reps, and the + // killswitch is not a standalone escape hatch. Keep AssemblyHelpers + // load/store/hasMegamorphicProperty and DFG/FTL compileHasOwnProperty in lockstep + // with this body (mov+mul64+urshift64, no tag test). + uint64_t bits = static_cast(reinterpret_cast(impl)); + return static_cast((bits * uidHashMultiplier) >> 32); +} + +ALWAYS_INLINE unsigned uidLength(const UniquedStringImpl* impl) +{ + if (isInlinePropertyKey(impl)) [[unlikely]] + return inlinePropertyKeyLength(inlinePropertyKeyWord(impl)); + return impl->length(); +} + +ALWAYS_INLINE void uidRef(UniquedStringImpl* impl) +{ + if (!(reinterpret_cast(impl) & inlinePropertyKeyTag)) [[likely]] + impl->ref(); +} + +ALWAYS_INLINE void uidDeref(UniquedStringImpl* impl) +{ + if (!(reinterpret_cast(impl) & inlinePropertyKeyTag)) [[likely]] + impl->deref(); +} + +// RefDerefTraits for RefPtr slots that may hold a +// fiber word instead of a real heap pointer. Matches DefaultRefDerefTraits +// shape so RefPtr/HashTable machinery works unchanged; only ref/deref branch. +struct FiberAwareRefDerefTraits { + static ALWAYS_INLINE UniquedStringImpl* refIfNotNull(UniquedStringImpl* ptr) + { + // Fuse the null-check and tag-check onto one register so clang emits + // test+jz; test+jnz with no redundant reload between them. + uintptr_t b = reinterpret_cast(ptr); + if (b && !(b & inlinePropertyKeyTag)) [[likely]] + ptr->ref(); + return ptr; + } + static ALWAYS_INLINE UniquedStringImpl& ref(UniquedStringImpl& r) + { + if (!(reinterpret_cast(&r) & inlinePropertyKeyTag)) [[likely]] + r.ref(); + return r; + } + static ALWAYS_INLINE void derefIfNotNull(UniquedStringImpl* ptr) + { + uintptr_t b = reinterpret_cast(ptr); + if (b && !(b & inlinePropertyKeyTag)) [[likely]] + ptr->deref(); + } +}; + +using FiberAwareRefPtr = RefPtr, FiberAwareRefDerefTraits>; +// PackedPtrTraits stores only EFFECTIVE_ADDRESS_WIDTH/8 = 6 bytes; a 6-7 char +// Latin-1 fiber word uses bytes 6-7, so Packed storage would truncate it. +// Alias to the full-width RawPtrTraits RefPtr and accept +2 bytes per entry. +using FiberAwarePackedRefPtr = FiberAwareRefPtr; + +} // namespace JSC + +#else // USE(BUN_JSC_ADDITIONS) + +namespace JSC { +// Transparent fallbacks so bare uidIsSymbol() / isInlinePropertyKey() at call +// sites compile in non-BUN builds without #if guards at each one. +ALWAYS_INLINE bool isInlinePropertyKey(const WTF::UniquedStringImpl*) { return false; } +ALWAYS_INLINE bool uidIsSymbol(const WTF::UniquedStringImpl* impl) { return impl->isSymbol(); } +} // namespace JSC + +#endif // USE(BUN_JSC_ADDITIONS) diff --git a/Source/JavaScriptCore/runtime/InlineStringCache.h b/Source/JavaScriptCore/runtime/InlineStringCache.h new file mode 100644 index 0000000000000..f128c3f786aef --- /dev/null +++ b/Source/JavaScriptCore/runtime/InlineStringCache.h @@ -0,0 +1,123 @@ +/* + * Copyright (C) 2024 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma once + +#if USE(BUN_JSC_ADDITIONS) + +#include +#include +#include +#include + +namespace JSC { + +class JSString; + +// Direct-mapped JSString* cache keyed by the inline-encoding fiber word. +// The fiber word is content-unique for 16-byte inline strings (lengths 2..7 +// Latin-1 / 1..3 UTF-16), so a single word compare is the full equality test. +// Weak: cleared at every GC finalize alongside KeyAtomStringCache. +class InlineStringCache { +public: + static constexpr unsigned capacity = 512; + + ALWAYS_INLINE JSString* lookup(uintptr_t fiber) const + { + unsigned index = indexFor(fiber); + if (m_keys[index] == fiber) + return m_values[index]; + return nullptr; + } + + ALWAYS_INLINE void insert(uintptr_t fiber, JSString* string) + { + unsigned index = indexFor(fiber); + m_keys[index] = fiber; + m_values[index] = string; + } + + ALWAYS_INLINE void clear() + { + m_keys.fill(0); + m_values.fill(nullptr); + } + +private: + ALWAYS_INLINE static unsigned indexFor(uintptr_t fiber) + { + // Low byte is flags+length; payload starts at byte 1. Fold the word + // once so chars 5..7 contribute to the index for ≤7-char Latin-1 keys. + uintptr_t mixed = fiber ^ (fiber >> 32); + return static_cast((mixed >> 8) % capacity); + } + + std::array m_keys { }; + std::array m_values { }; +}; + +// Direct-mapped AtomStringImpl* cache keyed by the inline fiber word, so +// Identifier::add/fromString for repeated short names skips the atom-table +// hash+probe. Holds a ref per slot; cleared on full GC. +class InlineAtomCache { +public: + static constexpr unsigned capacity = 512; + + ALWAYS_INLINE AtomStringImpl* lookup(uintptr_t fiber) const + { + unsigned index = indexFor(fiber); + if (m_keys[index] == fiber) + return m_values[index].get(); + return nullptr; + } + + ALWAYS_INLINE void insert(uintptr_t fiber, AtomStringImpl* atom) + { + unsigned index = indexFor(fiber); + m_keys[index] = fiber; + m_values[index] = atom; + } + + ALWAYS_INLINE void clear() + { + m_keys.fill(0); + for (auto& slot : m_values) + slot = nullptr; + } + +private: + ALWAYS_INLINE static unsigned indexFor(uintptr_t fiber) + { + uintptr_t mixed = fiber ^ (fiber >> 32); + return static_cast((mixed >> 8) % capacity); + } + + std::array m_keys { }; + std::array, capacity> m_values { }; +}; + +} // namespace JSC + +#endif // USE(BUN_JSC_ADDITIONS) diff --git a/Source/JavaScriptCore/runtime/JSCJSValue.cpp b/Source/JavaScriptCore/runtime/JSCJSValue.cpp index ee8dea1b312e3..352390f4b4d87 100644 --- a/Source/JavaScriptCore/runtime/JSCJSValue.cpp +++ b/Source/JavaScriptCore/runtime/JSCJSValue.cpp @@ -292,6 +292,10 @@ void JSValue::dumpInContextAssumingStructure( out.print("String"); if (string->isRope()) out.print(" (rope)"); +#if USE(BUN_JSC_ADDITIONS) + if (string->isInline()) + out.print(" (inline)"); +#endif const StringImpl* impl = string->tryGetValueImpl(); if (impl) { if (impl->isAtom()) diff --git a/Source/JavaScriptCore/runtime/JSFunction.cpp b/Source/JavaScriptCore/runtime/JSFunction.cpp index 748edf4d59e28..fef2b3005a144 100644 --- a/Source/JavaScriptCore/runtime/JSFunction.cpp +++ b/Source/JavaScriptCore/runtime/JSFunction.cpp @@ -186,6 +186,30 @@ FunctionRareData* JSFunction::initializeRareData(JSGlobalObject* globalObject, s return rareData; } +#if USE(BUN_JSC_ADDITIONS) +// Heap::runEndPhase nulls the thread AtomStringTable around finalizers, and +// ErrorInstance::finalizeUnconditionally builds stack traces there. For +// fiber-word Identifiers, Identifier::string() would materialize via +// AtomString(span) -> AtomStringImpl::add -> null table crash. Decode the +// payload to a plain String (heap copy, no atom table) instead. +static ALWAYS_INLINE String identifierToStringWithoutAtomizing(const Identifier& ident) +{ + UniquedStringImpl* uid = ident.impl(); + if (!uid) + return String(); + if (isInlinePropertyKey(uid)) { + uintptr_t word = inlinePropertyKeyWord(uid); + unsigned len = inlinePropertyKeyLength(word); + const uint8_t* bytes = reinterpret_cast(&word); + if (inlinePropertyKeyIs8Bit(word)) + return String(std::span { bytes + 1, len }); + return String(std::span { reinterpret_cast(bytes + 2), len }); + } + // Real UniquedStringImpl*: wrap directly, no atom-table lookup. + return String(static_cast(uid)); +} +#endif + String JSFunction::name(VM& vm) { if (isHostFunction()) { @@ -209,10 +233,17 @@ String JSFunction::nameWithoutGC(VM& vm) NativeExecutable* executable = uncheckedDowncast(this->executable()); return executable->name(); } +#if USE(BUN_JSC_ADDITIONS) + const Identifier& identifier = jsExecutable()->name(); + if (identifier == vm.propertyNames->starDefaultPrivateName) + return emptyString(); + return identifierToStringWithoutAtomizing(identifier); +#else const Identifier identifier = jsExecutable()->name(); if (identifier == vm.propertyNames->starDefaultPrivateName) return emptyString(); return identifier.string(); +#endif } String JSFunction::displayName(VM& vm) @@ -475,7 +506,13 @@ String getCalculatedDisplayName(VM& vm, JSObject* object) if (!actualName.isEmpty() || function->isHostOrBuiltinFunction()) return actualName; +#if USE(BUN_JSC_ADDITIONS) + // Runs off the mutator (see comment above) with the AtomStringTable nulled + // by Heap::runEndPhase; avoid Identifier::string()'s AtomString(span) path. + return identifierToStringWithoutAtomizing(function->jsExecutable()->ecmaName()); +#else return function->jsExecutable()->ecmaName().string(); +#endif } if (auto* function = dynamicDowncast(object)) return function->name(); diff --git a/Source/JavaScriptCore/runtime/JSGlobalObject.cpp b/Source/JavaScriptCore/runtime/JSGlobalObject.cpp index 565cd991263b7..1fef2ea5b5c6f 100644 --- a/Source/JavaScriptCore/runtime/JSGlobalObject.cpp +++ b/Source/JavaScriptCore/runtime/JSGlobalObject.cpp @@ -1484,7 +1484,11 @@ void JSGlobalObject::init(VM& vm) m_linkTimeConstants[static_cast(LinkTimeConstant::sentinelString)].set(vm, this, vm.smallStrings.sentinelString()); +#if USE(BUN_JSC_ADDITIONS) + JSFunction* defaultPromiseThen = JSFunction::create(vm, this, 2, vm.propertyNames->then.string(), promiseProtoFuncThen, ImplementationVisibility::Public, PromisePrototypeThenIntrinsic); +#else JSFunction* defaultPromiseThen = JSFunction::create(vm, this, 2, vm.propertyNames->then.impl(), promiseProtoFuncThen, ImplementationVisibility::Public, PromisePrototypeThenIntrinsic); +#endif m_linkTimeConstants[static_cast(LinkTimeConstant::defaultPromiseThen)].set(vm, this, defaultPromiseThen); #define CREATE_PROTOTYPE_FOR_SIMPLE_TYPE(capitalName, lowerName, properName, instanceType, jsName, prototypeBase, featureFlag) if (featureFlag) { \ diff --git a/Source/JavaScriptCore/runtime/JSGlobalObject.h b/Source/JavaScriptCore/runtime/JSGlobalObject.h index 486e6b583a625..5a6a1775ab45e 100644 --- a/Source/JavaScriptCore/runtime/JSGlobalObject.h +++ b/Source/JavaScriptCore/runtime/JSGlobalObject.h @@ -665,7 +665,11 @@ class JSGlobalObject : public JSSegmentedVariableObject { bool isArgumentsPrototypeIteratorProtocolFastAndNonObservable(); #if ENABLE(DFG_JIT) +#if USE(BUN_JSC_ADDITIONS) + using ReferencedGlobalPropertyWatchpointSets = UncheckedKeyHashMap, IdentifierRepHash>; +#else using ReferencedGlobalPropertyWatchpointSets = UncheckedKeyHashMap, Ref, IdentifierRepHash>; +#endif ReferencedGlobalPropertyWatchpointSets m_referencedGlobalPropertyWatchpointSets; ConcurrentJSLock m_referencedGlobalPropertyWatchpointSetsLock; #endif diff --git a/Source/JavaScriptCore/runtime/JSGlobalObjectFunctions.cpp b/Source/JavaScriptCore/runtime/JSGlobalObjectFunctions.cpp index 01b215d55d0e0..3be229a87c20e 100644 --- a/Source/JavaScriptCore/runtime/JSGlobalObjectFunctions.cpp +++ b/Source/JavaScriptCore/runtime/JSGlobalObjectFunctions.cpp @@ -41,6 +41,9 @@ #include "ObjectConstructorInlines.h" #include "ParseInt.h" #include "SourceProfiler.h" +#if USE(BUN_JSC_ADDITIONS) +#include "JSStringBuilder.h" +#endif #include #include #include @@ -68,7 +71,11 @@ static JSValue encode(JSGlobalObject* globalObject, const WTF::BitSet<256>& doNo return JSC::throwException(globalObject, scope, createURIError(globalObject, "String contained an illegal UTF-16 sequence."_s)); }; +#if USE(BUN_JSC_ADDITIONS) + JSStringBuilder builder(OverflowPolicy::RecordOverflow); +#else StringBuilder builder(OverflowPolicy::RecordOverflow); +#endif builder.reserveCapacity(characters.size()); // 4. Repeat @@ -134,7 +141,11 @@ static JSValue encode(JSGlobalObject* globalObject, const WTF::BitSet<256>& doNo if (builder.hasOverflowed()) [[unlikely]] return throwOutOfMemoryError(globalObject, scope); +#if USE(BUN_JSC_ADDITIONS) + return builder.toJS(vm); +#else return jsString(vm, builder.toString()); +#endif } static JSValue encode(JSGlobalObject* globalObject, JSValue argument, const WTF::BitSet<256>& doNotEscape) @@ -153,7 +164,11 @@ static JSValue decode(JSGlobalObject* globalObject, std::span ch VM& vm = globalObject->vm(); auto scope = DECLARE_THROW_SCOPE(vm); +#if USE(BUN_JSC_ADDITIONS) + JSStringBuilder builder(OverflowPolicy::RecordOverflow); +#else StringBuilder builder(OverflowPolicy::RecordOverflow); +#endif size_t k = 0; char16_t u = 0; while (k < characters.size()) { @@ -218,7 +233,11 @@ static JSValue decode(JSGlobalObject* globalObject, std::span ch } if (builder.hasOverflowed()) [[unlikely]] return throwOutOfMemoryError(globalObject, scope); +#if USE(BUN_JSC_ADDITIONS) + RELEASE_AND_RETURN(scope, builder.toJS(vm)); +#else RELEASE_AND_RETURN(scope, jsString(vm, builder.toString())); +#endif } static JSValue decode(JSGlobalObject* globalObject, JSValue argument, const WTF::BitSet<256>& doNotUnescape, bool strict) @@ -614,7 +633,11 @@ JSC_DEFINE_HOST_FUNCTION(globalFuncEscape, (JSGlobalObject* globalObject, CallFr VM& vm = globalObject->vm(); auto scope = DECLARE_THROW_SCOPE(vm); +#if USE(BUN_JSC_ADDITIONS) + JSStringBuilder builder(OverflowPolicy::RecordOverflow); +#else StringBuilder builder(OverflowPolicy::RecordOverflow); +#endif if (view.is8Bit()) { for (auto character : view.span8()) { if (doNotEscape.get(character)) @@ -637,7 +660,11 @@ JSC_DEFINE_HOST_FUNCTION(globalFuncEscape, (JSGlobalObject* globalObject, CallFr throwOutOfMemoryError(globalObject, scope); return { }; } +#if USE(BUN_JSC_ADDITIONS) + return builder.toJS(vm); +#else return jsString(vm, builder.toString()); +#endif })); } @@ -652,7 +679,11 @@ JSC_DEFINE_HOST_FUNCTION(globalFuncUnescape, (JSGlobalObject* globalObject, Call VM& vm = globalObject->vm(); auto scope = DECLARE_THROW_SCOPE(vm); +#if USE(BUN_JSC_ADDITIONS) + JSStringBuilder builder(OverflowPolicy::RecordOverflow); +#else StringBuilder builder(OverflowPolicy::RecordOverflow); +#endif builder.reserveCapacity(length); if (view.is8Bit()) { @@ -700,7 +731,11 @@ JSC_DEFINE_HOST_FUNCTION(globalFuncUnescape, (JSGlobalObject* globalObject, Call throwOutOfMemoryError(globalObject, scope); return { }; } +#if USE(BUN_JSC_ADDITIONS) + return builder.toJS(vm); +#else return jsString(vm, builder.toString()); +#endif })); } diff --git a/Source/JavaScriptCore/runtime/JSGlobalObjectInlines.h b/Source/JavaScriptCore/runtime/JSGlobalObjectInlines.h index 7a81f7901af37..d4848c3ab8363 100644 --- a/Source/JavaScriptCore/runtime/JSGlobalObjectInlines.h +++ b/Source/JavaScriptCore/runtime/JSGlobalObjectInlines.h @@ -271,9 +271,15 @@ template template inline unsigned JSGlobalObject::WeakCustomGetterOrSetterHash::hash(const PropertyName& propertyName, typename U::CustomFunctionPointer functionPointer, const ClassInfo* classInfo) { +#if USE(BUN_JSC_ADDITIONS) + if (!propertyName.isNull()) + return WTF::computeHash(functionPointer, uidHash(propertyName.uid()), classInfo); + return WTF::computeHash(functionPointer, classInfo); +#else if (!propertyName.isNull()) return WTF::computeHash(functionPointer, propertyName.uid()->existingSymbolAwareHash(), classInfo); return WTF::computeHash(functionPointer, classInfo); +#endif } inline JSArray* constructEmptyArray(JSGlobalObject* globalObject, ArrayAllocationProfile* profile, unsigned initialLength = 0, JSValue newTarget = JSValue()) diff --git a/Source/JavaScriptCore/runtime/JSLexicalEnvironment.cpp b/Source/JavaScriptCore/runtime/JSLexicalEnvironment.cpp index 11842faf429c5..89a36a52abba6 100644 --- a/Source/JavaScriptCore/runtime/JSLexicalEnvironment.cpp +++ b/Source/JavaScriptCore/runtime/JSLexicalEnvironment.cpp @@ -81,8 +81,13 @@ void JSLexicalEnvironment::getOwnSpecialPropertyNames(JSObject* object, JSGlobal continue; if (!thisObject->isValidScopeOffset(it->value.scopeOffset())) continue; +#if USE(BUN_JSC_ADDITIONS) + if (!propertyNames.includeSymbolProperties() && uidIsSymbol(it->key.get())) + continue; +#else if (!propertyNames.includeSymbolProperties() && it->key->isSymbol()) continue; +#endif if (propertyNames.privateSymbolMode() == PrivateSymbolMode::Exclude && symbolTable->hasPrivateName(it->key)) continue; propertyNames.add(Identifier::fromUid(vm, it->key.get())); diff --git a/Source/JavaScriptCore/runtime/JSModuleNamespaceObject.cpp b/Source/JavaScriptCore/runtime/JSModuleNamespaceObject.cpp index 18c7e9afd5e31..0aefad406c034 100644 --- a/Source/JavaScriptCore/runtime/JSModuleNamespaceObject.cpp +++ b/Source/JavaScriptCore/runtime/JSModuleNamespaceObject.cpp @@ -49,7 +49,15 @@ JSModuleNamespaceObject::JSModuleNamespaceObject(VM& vm, Structure* structure, A // // Sort the exported names by the code point order. std::ranges::sort(resolutions, WTF::codePointCompareLessThan, [](const auto& resolution) { +#if USE(BUN_JSC_ADDITIONS) + // Identifier::string() now returns AtomString by value, so wrapping it in + // StringView would dangle once the lambda returns. Return an owning String + // instead; std::ranges::sort keeps both projected temporaries alive across + // comp(proj(a), proj(b)), and String implicitly converts to StringView. + return resolution.first.stringWithoutAtomizing(); +#else return StringView(resolution.first.impl()); +#endif }); for (const auto& pair : resolutions) { diff --git a/Source/JavaScriptCore/runtime/JSModuleNamespaceObject.h b/Source/JavaScriptCore/runtime/JSModuleNamespaceObject.h index 9e3e15b746a75..7434faa4a825c 100644 --- a/Source/JavaScriptCore/runtime/JSModuleNamespaceObject.h +++ b/Source/JavaScriptCore/runtime/JSModuleNamespaceObject.h @@ -90,7 +90,11 @@ class JSModuleNamespaceObject final : public JSNonFinalObject { WriteBarrier moduleRecord; }; +#if USE(BUN_JSC_ADDITIONS) + using ExportMap = OrderedHashMap>; +#else using ExportMap = OrderedHashMap, ExportEntry, IdentifierRepHash, HashTraits>>; +#endif ExportMap m_exports; WriteBarrier m_moduleRecord; diff --git a/Source/JavaScriptCore/runtime/JSONObject.cpp b/Source/JavaScriptCore/runtime/JSONObject.cpp index fc398dbaa78b4..0cfa275c563ce 100644 --- a/Source/JavaScriptCore/runtime/JSONObject.cpp +++ b/Source/JavaScriptCore/runtime/JSONObject.cpp @@ -221,9 +221,17 @@ inline PropertyNameForFunctionCall::PropertyNameForFunctionCall(unsigned number) JSValue PropertyNameForFunctionCall::value(VM& vm) const { if (!m_value) { - if (!m_propertyName.isNull()) + if (!m_propertyName.isNull()) { +#if USE(BUN_JSC_ADDITIONS) + UniquedStringImpl* uid = m_propertyName.uid(); + if (isInlinePropertyKey(uid)) + m_value = JSString::createInlineFromFiber(vm, inlinePropertyKeyWord(uid)); + else + m_value = jsString(vm, String { uid }); +#else m_value = jsString(vm, String { m_propertyName.uid() }); - else { +#endif + } else { if (m_number <= 9) return vm.smallStrings.singleCharacterString(m_number + '0'); m_value = jsNontrivialString(vm, vm.numericStrings.add(m_number)); @@ -640,7 +648,18 @@ bool Stringifier::Holder::appendNextProperty(Stringifier& stringifier, StringBui stringifier.startNewLine(builder); // Append the property name, colon, and space. +#if USE(BUN_JSC_ADDITIONS) + { + UniquedStringImpl* uid = propertyName.uid(); + if (isInlinePropertyKey(uid)) { + uintptr_t word = inlinePropertyKeyWord(uid); + builder.appendQuotedJSONString(String(inlinePropertyKeySpan8(word))); + } else + builder.appendQuotedJSONString(*uid); + } +#else builder.appendQuotedJSONString(*propertyName.uid()); +#endif builder.append(':'); if (stringifier.willIndent()) builder.append(' '); @@ -1426,6 +1445,29 @@ void FastStringifier::append(JSValue value) structure.forEachProperty(m_vm, [&](const auto& entry) -> bool { if (entry.attributes() & PropertyAttribute::DontEnum) return true; +#if USE(BUN_JSC_ADDITIONS) + UniquedStringImpl* nameUID = entry.key(); + uintptr_t nameWord = inlinePropertyKeyWord(nameUID); + std::span span; + if (isInlinePropertyKey(nameUID)) { + if (!inlinePropertyKeyIs8Bit(nameWord)) [[unlikely]] { + recordFailure("16-bit property name"_s); + return false; + } + span = inlinePropertyKeySpan8(nameWord); + } else { + auto& name = *nameUID; + if (name.isSymbol()) [[unlikely]] { + recordFailure("symbol"_s); + return false; + } + if (!name.is8Bit()) [[unlikely]] { + recordFailure("16-bit property name"_s); + return false; + } + span = name.span8(); + } +#else auto& name = *entry.key(); if (name.isSymbol()) [[unlikely]] { recordFailure("symbol"_s); @@ -1439,6 +1481,7 @@ void FastStringifier::append(JSValue value) return false; } auto span = name.span8(); +#endif // The structure cannot transition mid-iteration on FastStringifier's case. // canPerformFastPropertyEnumeration ruled out getters/setters and diff --git a/Source/JavaScriptCore/runtime/JSPropertyNameEnumerator.cpp b/Source/JavaScriptCore/runtime/JSPropertyNameEnumerator.cpp index 5f95ef542ee0a..d2027e751f80b 100644 --- a/Source/JavaScriptCore/runtime/JSPropertyNameEnumerator.cpp +++ b/Source/JavaScriptCore/runtime/JSPropertyNameEnumerator.cpp @@ -27,6 +27,10 @@ #include "JSPropertyNameEnumerator.h" #include "JSObjectInlines.h" +#if USE(BUN_JSC_ADDITIONS) +#include "IdentifierInlines.h" +#include "InlinePropertyKey.h" +#endif WTF_ALLOW_UNSAFE_BUFFER_USAGE_BEGIN @@ -81,6 +85,9 @@ void JSPropertyNameEnumerator::finishCreation(VM& vm, RefPtr& ASSERT(m_endGenericPropertyIndex == vector.size()); for (unsigned i = 0; i < vector.size(); ++i) { const Identifier& identifier = vector[i]; + // audit-string-materialize: KEEP atom-backed here. for-in keys flow into + // user JS where DFG StringCharAt / substring / length inline the m_fiber + // → StringImpl::m_length load without an inline-cell guard. m_propertyNames.get()[i].set(vm, this, jsString(vm, identifier.string())); } } @@ -204,8 +211,19 @@ JSString* JSPropertyNameEnumerator::computeNext(JSGlobalObject* globalObject, JS break; auto id = name->toAtomString(globalObject); RETURN_IF_EXCEPTION(scope, nullptr); +#if USE(BUN_JSC_ADDITIONS) + // D.4 coherence: id.data is an AtomStringImpl*; Structure::get's + // seenProperties bloom filter was keyed on the canonical fiber word + // at add() time, so re-encode here or the bloom filter rules us out. + UniquedStringImpl* uid = id.data; + if (uintptr_t fiber = Identifier::canonicalFiberWordFor(uid)) + uid = reinterpret_cast(fiber); + if (base->hasEnumerableProperty(globalObject, uid)) + break; +#else if (base->hasEnumerableProperty(globalObject, id.data)) break; +#endif RETURN_IF_EXCEPTION(scope, nullptr); name = nullptr; index++; diff --git a/Source/JavaScriptCore/runtime/JSScope.h b/Source/JavaScriptCore/runtime/JSScope.h index be8fd8d57b6da..3a596b1e8d1a3 100644 --- a/Source/JavaScriptCore/runtime/JSScope.h +++ b/Source/JavaScriptCore/runtime/JSScope.h @@ -35,7 +35,11 @@ class ScopeChainIterator; class SymbolTable; class WatchpointSet; +#if USE(BUN_JSC_ADDITIONS) +// TDZEnvironment is defined in VariableEnvironment.h with FiberAwareRefPtr; avoid a conflicting redefinition here. +#else using TDZEnvironment = UncheckedKeyHashSet, IdentifierRepHash>; +#endif class JSScope : public JSNonFinalObject { public: diff --git a/Source/JavaScriptCore/runtime/JSString.cpp b/Source/JavaScriptCore/runtime/JSString.cpp index 244b58c5102d3..2ab05dca09ecc 100644 --- a/Source/JavaScriptCore/runtime/JSString.cpp +++ b/Source/JavaScriptCore/runtime/JSString.cpp @@ -75,6 +75,10 @@ void JSString::dumpToStream(const JSCell* cell, PrintStream& out) out.printf("[substring]"); else out.printf("[rope]"); +#if USE(BUN_JSC_ADDITIONS) + } else if (isInlineFiber(pointer)) { + out.printf("[inline %s]", (pointer & JSRopeString::is8BitInPointer) ? "8" : "16"); +#endif } else { if (WTF::StringImpl* ourImpl = std::bit_cast(pointer)) { if (ourImpl->is8Bit()) @@ -95,7 +99,7 @@ size_t JSString::estimatedSize(JSCell* cell, VM& vm) { JSString* thisObject = asString(cell); uintptr_t pointer = thisObject->fiberConcurrently(); - if (pointer & isRopeInPointer) + if (pointer & notStringImplMask) return Base::estimatedSize(cell, vm); return Base::estimatedSize(cell, vm) + std::bit_cast(pointer)->costDuringGC(); } @@ -135,10 +139,75 @@ void JSString::visitChildrenImpl(JSCell* cell, Visitor& visitor) } return; } +#if USE(BUN_JSC_ADDITIONS) + if (isInlineFiber(pointer)) + return; +#endif if (StringImpl* impl = std::bit_cast(pointer)) visitor.reportExtraMemoryVisited(impl->costDuringGC()); } +#if USE(BUN_JSC_ADDITIONS) +const String& JSString::resolveInline(JSGlobalObject* globalObject) const +{ + ASSERT(isInline()); + uintptr_t fiber = m_fiber; + unsigned len = inlineLengthFromFiber(fiber); + String result = (fiber & JSRopeString::is8BitInPointer) + ? String(StringImpl::create(std::span { inlineData8(), len })) + : String(StringImpl::create(std::span { inlineData16(), len })); + WTF::storeStoreFence(); + new (&uninitializedValueInternal()) String(WTF::move(result)); + if (globalObject) + getVM(globalObject).heap.reportExtraMemoryAllocated(this, valueInternal().impl()->cost()); + return valueInternal(); +} + +AtomStringImpl* JSString::resolveInlineToAtomString(JSGlobalObject* globalObject) const +{ + ASSERT(isInline()); + uintptr_t fiber = m_fiber; + unsigned len = inlineLengthFromFiber(fiber); + bool is8Bit = fiber & JSRopeString::is8BitInPointer; + // 16-byte inline: consult the fiber-word-keyed atom cache first. + VM& vm = globalObject ? getVM(globalObject) : this->vm(); + if ((is8Bit && len <= maxInlineLength8) || (!is8Bit && len <= maxInlineLength16)) { + if (auto* cached = vm.inlineAtomCache.lookup(fiber)) { + WTF::storeStoreFence(); + new (&uninitializedValueInternal()) String(RefPtr { cached }); + return cached; + } + } + // Look up (or create) the atom directly from the in-cell bytes; no + // intermediate non-atom StringImpl is allocated. + AtomString atom = is8Bit + ? AtomString(std::span { inlineData8(), len }) + : AtomString(std::span { inlineData16(), len }); + if ((is8Bit && len <= maxInlineLength8) || (!is8Bit && len <= maxInlineLength16)) + vm.inlineAtomCache.insert(fiber, atom.impl()); + size_t sizeToReport = atom.impl()->hasOneRef() ? atom.impl()->cost() : 0; + WTF::storeStoreFence(); + new (&uninitializedValueInternal()) String(atom.releaseImpl()); + vm.heap.reportExtraMemoryAllocated(this, sizeToReport); + return static_cast(valueInternal().impl()); +} + +AtomStringImpl* JSString::resolveInlineToExistingAtomString() const +{ + ASSERT(isInline()); + uintptr_t fiber = m_fiber; + unsigned len = inlineLengthFromFiber(fiber); + RefPtr atom = (fiber & JSRopeString::is8BitInPointer) + ? AtomStringImpl::lookUp(std::span { inlineData8(), len }) + : AtomStringImpl::lookUp(std::span { inlineData16(), len }); + if (!atom) + return nullptr; + WTF::storeStoreFence(); + new (&uninitializedValueInternal()) String(WTF::move(atom)); + return static_cast(valueInternal().impl()); +} +#endif + DEFINE_VISIT_CHILDREN(JSString); template @@ -349,6 +418,10 @@ GCOwnedDataScope JSString::tryGetValueWithoutGC() const // Pass nullptr for the JSGlobalObject so that resolveRope does not throw in the event of an OOM error. return { this, static_cast(this)->resolveRopeWithoutGC() }; } +#if USE(BUN_JSC_ADDITIONS) + if (isInline()) + return { this, resolveInline(nullptr) }; +#endif return { this, valueInternal() }; } diff --git a/Source/JavaScriptCore/runtime/JSString.h b/Source/JavaScriptCore/runtime/JSString.h index fddac10ab622e..0ee1329d80944 100644 --- a/Source/JavaScriptCore/runtime/JSString.h +++ b/Source/JavaScriptCore/runtime/JSString.h @@ -79,6 +79,17 @@ JSString* jsNontrivialString(VM&, String&&); // DOM object that contains a String JSString* jsOwnedString(VM&, const String&); +#if USE(BUN_JSC_ADDITIONS) +// Build a JSString for a property key that may be a fiber-word-tagged +// UniquedStringImpl* (Identifier::impl() / PropertyName::uid()). Fiber words +// share JSString's inline m_fiber layout, so they flow straight into +// createInlineFromFiber — no AtomStringTable round-trip. Real impls are +// interned atoms owned elsewhere, so take the createHasOtherOwner path. +JSString* jsStringFromFiberOrImpl(VM&, UniquedStringImpl*); +JSString* jsStringFromFiberOrImpl(VM&, const Identifier&); +JSString* jsOwnedAtomBackedString(VM&, const Identifier&); +#endif + bool isJSString(JSCell*); bool isJSString(JSValue); JSString* asString(JSValue); @@ -138,6 +149,25 @@ class JSString : public JSCell { static constexpr unsigned minLengthForRopeWalk = 0x128; static constexpr uintptr_t isRopeInPointer = 0x1; +#if USE(BUN_JSC_ADDITIONS) + // Inline small strings: bit 1 of m_fiber set with bit 0 clear means the + // remaining bytes of m_fiber hold the character data directly (no + // StringImpl). Bit 2 is the is8Bit flag (same bit JSRopeString uses). + static constexpr uintptr_t isInlineInPointer = 0x2; + static constexpr uintptr_t notStringImplMask = isRopeInPointer | isInlineInPointer; + static constexpr unsigned inlineLengthShift = 3; + // 4-bit mask covers both the 16-byte JSString inline (len 2..7) and the + // 24-byte JSBigInlineString (len 8..15). Payload is always contiguous + // starting at &m_fiber+1; the cell size is determined at creation. + static constexpr unsigned inlineLengthMask = 0xf; + static constexpr unsigned maxInlineLength8 = 7; + static constexpr unsigned maxInlineLength16 = 3; + static constexpr unsigned maxBigInlineLength8 = 15; + static constexpr unsigned maxBigInlineLength16 = 7; + static_assert(maxBigInlineLength8 <= inlineLengthMask); +#else + static constexpr uintptr_t notStringImplMask = isRopeInPointer; +#endif static constexpr unsigned maxLengthForOnStackResolve = 2048; @@ -152,7 +182,7 @@ class JSString : public JSCell { String& valueInternal() const { - ASSERT(!isRope()); + ASSERT(!(m_fiber & notStringImplMask)); return uninitializedValueInternal(); } @@ -174,6 +204,18 @@ class JSString : public JSCell { { } +#if USE(BUN_JSC_ADDITIONS) +protected: + enum InlineTag { CreateInline }; + JSString(VM& vm, InlineTag, uintptr_t encodedFiber) + : JSCell(CreatingWellDefinedBuiltinCell, vm.stringStructure.get()->id(), defaultTypeInfoBlob()) + , m_fiber(encodedFiber) + { + } +#endif + +#define BUN_INLINE_COUNT(name) ((void)0) + void finishCreation(VM& vm, unsigned length) { ASSERT_UNUSED(length, length > 0); @@ -201,6 +243,7 @@ class JSString : public JSCell { static JSString* create(VM& vm, Ref&& value) { + BUN_INLINE_COUNT(g_bunJSStringCreate); unsigned length = value->length(); ASSERT(length > 0); size_t cost = value->cost(); @@ -226,6 +269,51 @@ class JSString : public JSCell { return newString; } +#if USE(BUN_JSC_ADDITIONS) +public: + static ALWAYS_INLINE uintptr_t encodeInline8(std::span chars) + { + ASSERT(chars.size() >= 2 && chars.size() <= maxInlineLength8); + uintptr_t fiber = isInlineInPointer | static_cast(StringImpl::flagIs8Bit()) + | (static_cast(chars.size()) << inlineLengthShift); + memcpy(reinterpret_cast(&fiber) + 1, chars.data(), chars.size()); + return fiber; + } + + static ALWAYS_INLINE uintptr_t encodeInline16(std::span chars) + { + ASSERT(chars.size() >= 1 && chars.size() <= maxInlineLength16); + uintptr_t fiber = isInlineInPointer + | (static_cast(chars.size()) << inlineLengthShift); + memcpy(reinterpret_cast(&fiber) + 2, chars.data(), chars.size() * sizeof(char16_t)); + return fiber; + } + + static JSString* createInline8(VM& vm, std::span chars); + static JSString* createInline16(VM& vm, std::span chars); + static JSString* createInline8(VM& vm, GCDeferralContext*, std::span chars); + static JSString* createInline16(VM& vm, GCDeferralContext*, std::span chars); + + static ALWAYS_INLINE JSString* createInlineFromFiber(VM& vm, uintptr_t fiber) + { + BUN_INLINE_COUNT(g_bunInlineCreated); + JSString* s = new (NotNull, allocateCell(vm)) JSString(vm, CreateInline, fiber); + s->Base::finishCreation(vm); + return s; + } + + // jsSubstringOfResolved is called with a live GCDeferralContext while + // RegExpMatchesArray slots are still uninitialised; route allocations + // through it so a slow-path GC defers instead of visiting garbage. + static ALWAYS_INLINE JSString* createInlineFromFiber(VM& vm, GCDeferralContext* deferralContext, uintptr_t fiber) + { + BUN_INLINE_COUNT(g_bunInlineCreated); + JSString* s = new (NotNull, allocateCell(vm, deferralContext)) JSString(vm, CreateInline, fiber); + s->Base::finishCreation(vm); + return s; + } +#endif + protected: DECLARE_DEFAULT_FINISH_CREATION; @@ -271,6 +359,38 @@ class JSString : public JSCell { { return m_fiber & isRopeInPointer; } + +#if USE(BUN_JSC_ADDITIONS) + ALWAYS_INLINE bool isInline() const + { + return (fiberConcurrently() & notStringImplMask) == isInlineInPointer; + } + ALWAYS_INLINE uintptr_t inlineFiberWord() const + { + ASSERT(isInline()); + return fiberConcurrently(); + } + ALWAYS_INLINE uintptr_t tryGetCanonicalInlineFiberWord() const; + ALWAYS_INLINE static bool isInlineFiber(uintptr_t fiber) + { + return (fiber & notStringImplMask) == isInlineInPointer; + } + ALWAYS_INLINE static unsigned inlineLengthFromFiber(uintptr_t fiber) + { + return static_cast(fiber >> inlineLengthShift) & inlineLengthMask; + } + ALWAYS_INLINE const Latin1Character* inlineData8() const + { + return reinterpret_cast(&m_fiber) + 1; + } + ALWAYS_INLINE const char16_t* inlineData16() const + { + return reinterpret_cast(reinterpret_cast(&m_fiber) + 2); + } + JS_EXPORT_PRIVATE const String& resolveInline(JSGlobalObject*) const; + JS_EXPORT_PRIVATE AtomStringImpl* resolveInlineToAtomString(JSGlobalObject*) const; + JS_EXPORT_PRIVATE AtomStringImpl* resolveInlineToExistingAtomString() const; +#endif ALWAYS_INLINE JSRopeString* asRope() { ASSERT(isRope()); @@ -330,11 +450,78 @@ class JSString : public JSCell { friend JSString* tryJSSubstringImpl(VM&, JSString*, unsigned, unsigned); friend JSString* jsSubstringOfResolved(VM&, GCDeferralContext*, JSString*, unsigned, unsigned); friend JSString* jsOwnedString(VM&, const String&); +#if USE(BUN_JSC_ADDITIONS) + friend JSString* jsStringFromFiberOrImpl(VM&, UniquedStringImpl*); + friend JSString* jsOwnedAtomBackedString(VM&, const Identifier&); +#endif friend JSString* jsAtomString(JSGlobalObject*, VM&, JSString*); friend JSString* jsAtomString(JSGlobalObject*, VM&, JSString*, JSString*); friend JSString* jsAtomString(JSGlobalObject*, VM&, JSString*, JSString*, JSString*); }; +#if USE(BUN_JSC_ADDITIONS) +// 24-byte inline string: same encoding as JSString's inline variant, with an +// extra 8 bytes of contiguous payload so lengths 8..15 (Latin-1) / 4..7 +// (UTF-16) fit without a StringImpl. inlineData8()/inlineData16() and every +// JIT decoder already index from &m_fiber+1, so they work unchanged. +class JSBigInlineString final : public JSString { + friend class JSString; +public: + using Base = JSString; + static constexpr DestructionMode needsDestruction = MayNeedDestruction; + static constexpr uint8_t numberOfLowerTierPreciseCells = 0; + + template + static GCClient::IsoSubspace* subspaceFor(VM& vm) + { + return &vm.bigInlineStringSpace(); + } + + // After resolveInline() this holds a StringImpl*; reuse JSString::destroy + // which already checks notStringImplMask. + static void destroy(JSCell* cell) { JSString::destroy(cell); } + + static JSBigInlineString* create8(VM& vm, std::span chars) { return create8(vm, nullptr, chars); } + static JSBigInlineString* create8(VM& vm, GCDeferralContext* deferralContext, std::span chars) + { + BUN_INLINE_COUNT(g_bunBigInlineCreated); + ASSERT(chars.size() > maxInlineLength8 && chars.size() <= maxBigInlineLength8); + JSBigInlineString* s = new (NotNull, allocateCell(vm, deferralContext)) JSBigInlineString(vm); + uint8_t* bytes = reinterpret_cast(&s->m_fiber); + bytes[0] = static_cast(isInlineInPointer | StringImpl::flagIs8Bit() + | (chars.size() << inlineLengthShift)); + memcpy(bytes + 1, chars.data(), chars.size()); + s->finishCreation(vm); + return s; + } + + static JSBigInlineString* create16(VM& vm, std::span chars) { return create16(vm, nullptr, chars); } + static JSBigInlineString* create16(VM& vm, GCDeferralContext* deferralContext, std::span chars) + { + BUN_INLINE_COUNT(g_bunBigInlineCreated); + ASSERT(chars.size() > maxInlineLength16 && chars.size() <= maxBigInlineLength16); + JSBigInlineString* s = new (NotNull, allocateCell(vm, deferralContext)) JSBigInlineString(vm); + uint8_t* bytes = reinterpret_cast(&s->m_fiber); + bytes[0] = static_cast(isInlineInPointer | (chars.size() << inlineLengthShift)); + bytes[1] = 0; + memcpy(bytes + 2, chars.data(), chars.size() * sizeof(char16_t)); + s->finishCreation(vm); + return s; + } + +private: + JSBigInlineString(VM& vm) + : JSString(vm, CreateInline, 0) + , m_inlinePayloadHigh(0) + { } + + DECLARE_DEFAULT_FINISH_CREATION; + + uintptr_t m_inlinePayloadHigh; +}; +static_assert(sizeof(JSBigInlineString) == 24); +#endif + // NOTE: This class cannot override JSString's destructor. JSString's destructor is called directly // from JSStringSubspace:: class JSRopeString final : public JSString { @@ -648,6 +835,7 @@ class JSRopeString final : public JSString { static JSRopeString* create(VM& vm, JSString* s1, JSString* s2) { + BUN_INLINE_COUNT(g_bunRopeCreated); unsigned length = s1->length() + s2->length(); bool is8Bit = !!(static_cast(!!s1->is8Bit()) & static_cast(!!s2->is8Bit())); JSRopeString* newString = new (NotNull, allocateCell(vm)) JSRopeString(vm, length, is8Bit, s1, s2); @@ -658,6 +846,7 @@ class JSRopeString final : public JSString { } static JSRopeString* create(VM& vm, JSString* s1, JSString* s2, JSString* s3) { + BUN_INLINE_COUNT(g_bunRopeCreated); unsigned length = s1->length() + s2->length() + s3->length(); bool is8Bit = !!(static_cast(!!s1->is8Bit()) & static_cast(!!s2->is8Bit()) & static_cast(!!s3->is8Bit())); JSRopeString* newString = new (NotNull, allocateCell(vm)) JSRopeString(vm, length, is8Bit, s1, s2, s3); @@ -669,6 +858,7 @@ class JSRopeString final : public JSString { ALWAYS_INLINE static JSRopeString* createSubstringOfResolved(VM& vm, GCDeferralContext* deferralContext, JSString* base, unsigned offset, unsigned length, bool is8Bit) { + BUN_INLINE_COUNT(g_bunRopeSubstringCreated); JSRopeString* newString = new (NotNull, allocateCell(vm, deferralContext)) JSRopeString(vm, length, is8Bit, base, offset); newString->finishCreationSubstringOfResolved(vm); ASSERT(newString->length()); @@ -784,9 +974,10 @@ JS_EXPORT_PRIVATE JSString* jsStringWithCacheSlowCase(VM&, StringImpl&); ALWAYS_INLINE bool JSString::is8Bit() const { uintptr_t pointer = fiberConcurrently(); - if (pointer & isRopeInPointer) { + if (pointer & notStringImplMask) { // Do not load m_fiber twice. We should use the information in pointer. // Otherwise, JSRopeString may be converted to JSString between the first and second accesses. + // Rope and inline both encode is8Bit in the same low bit. return pointer & JSRopeString::is8BitInPointer; } return std::bit_cast(pointer)->is8Bit(); @@ -798,21 +989,29 @@ ALWAYS_INLINE bool JSString::is8Bit() const ALWAYS_INLINE unsigned JSString::length() const { uintptr_t pointer = fiberConcurrently(); +#if USE(BUN_JSC_ADDITIONS) + if (pointer & notStringImplMask) { + if (pointer & isRopeInPointer) + return uncheckedDowncast(this)->length(); + return inlineLengthFromFiber(pointer); + } +#else if (pointer & isRopeInPointer) return uncheckedDowncast(this)->length(); +#endif return std::bit_cast(pointer)->length(); } inline StringImpl* JSString::getValueImpl() const { - ASSERT(!isRope()); + ASSERT(!(m_fiber & notStringImplMask)); return std::bit_cast(m_fiber); } inline StringImpl* JSString::tryGetValueImpl() const { uintptr_t pointer = fiberConcurrently(); - if (pointer & isRopeInPointer) + if (pointer & notStringImplMask) return nullptr; return std::bit_cast(pointer); } @@ -884,12 +1083,45 @@ ALWAYS_INLINE void JSString::swapToAtomString(VM& vm, RefPtr&& a vm.heap.appendPossiblyAccessedStringFromConcurrentThreadsOrGCOwnedDataScope(this, WTF::move(target)); } +#if USE(BUN_JSC_ADDITIONS) +// D.4 fast path: when this cell is an 8-bit small-inline of length 2..5, m_fiber +// IS the canonical fiber-word key. Returning it lets the JIT-operation slow paths +// (getByVal*/putByVal*/HasOwnProperty) skip the resolveInlineToAtomString() + +// canonicalFiberWordFor() round-trip, and — critically — leaves m_fiber unmutated +// so the next JIT fast-path probe for this cell (and any other cell with the same +// content) sees the same bits and hits the MegamorphicCache/HasOwnPropertyCache. +ALWAYS_INLINE uintptr_t JSString::tryGetCanonicalInlineFiberWord() const +{ + if constexpr (!enableIdentifierFiberWords) + return 0; + uintptr_t f = fiberConcurrently(); + if ((f & (notStringImplMask | JSRopeString::is8BitInPointer)) != (isInlineInPointer | JSRopeString::is8BitInPointer)) + return 0; + if (inlineLengthFromFiber(f) > maxFiberWordKeyLength) + return 0; + return f; +} +#endif + ALWAYS_INLINE Identifier JSString::toIdentifier(JSGlobalObject* globalObject) const { if constexpr (validateDFGDoesGC) getVM(globalObject).verifyCanGC(); if (isRope()) return static_cast(this)->toIdentifier(globalObject); +#if USE(BUN_JSC_ADDITIONS) + if (isInline()) { + // Phase D.2 producer + D.4 coherence: only the 8-bit small-inline + // encoding is the canonical Identifier form. 16-bit inline and + // big-inline (8..15) atomize, then fromString() re-canonicalizes + // to encodeInline8 if the content is 2..5 Latin-1. + if constexpr (enableIdentifierFiberWords) { + if (is8Bit() && length() <= maxFiberWordKeyLength) + return Identifier::fromFiberWord(m_fiber); + } + return Identifier::fromString(getVM(globalObject), Ref { *resolveInlineToAtomString(globalObject) }); + } +#endif VM& vm = getVM(globalObject); if (valueInternal().impl()->isAtom()) return Identifier::fromString(vm, Ref { *static_cast(valueInternal().impl()) }); @@ -910,6 +1142,10 @@ ALWAYS_INLINE GCOwnedDataScope JSString::toAtomString(JSGlobalO getVM(globalObject).verifyCanGC(); if (isRope()) return { this, static_cast(this)->resolveRopeToAtomString(globalObject) }; +#if USE(BUN_JSC_ADDITIONS) + if (isInline()) + return { this, resolveInlineToAtomString(globalObject) }; +#endif if (valueInternal().impl()->isAtom()) return { this, static_cast(valueInternal().impl()) }; AtomString atom(valueInternal()); @@ -923,6 +1159,13 @@ ALWAYS_INLINE GCOwnedDataScope JSString::toExistingAtomString(J getVM(globalObject).verifyCanGC(); if (isRope()) return static_cast(this)->resolveRopeToExistingAtomString(globalObject); +#if USE(BUN_JSC_ADDITIONS) + if (isInline()) { + if (auto* atom = resolveInlineToExistingAtomString()) + return { this, atom }; + return { }; + } +#endif if (valueInternal().impl()->isAtom()) return { this, static_cast(valueInternal().impl()) }; if (auto atom = AtomStringImpl::lookUp(valueInternal().impl())) { @@ -938,6 +1181,10 @@ inline GCOwnedDataScope JSString::value(JSGlobalObject* globalObj getVM(globalObject).verifyCanGC(); if (isRope()) return { this, static_cast(this)->resolveRope(globalObject) }; +#if USE(BUN_JSC_ADDITIONS) + if (isInline()) + return { this, resolveInline(globalObject) }; +#endif return { this, valueInternal() }; } #if USE(BUN_JSC_ADDITIONS) @@ -947,6 +1194,15 @@ inline void JSString::value(jsstring_iterator* iterator) const static_cast(this)->iterRope(iterator); return; } + uintptr_t fiber = fiberConcurrently(); + if (isInlineFiber(fiber)) { + unsigned len = inlineLengthFromFiber(fiber); + if (fiber & JSRopeString::is8BitInPointer) + iterator->append8(iterator, (void*)inlineData8(), len); + else + iterator->append16(iterator, (void*)inlineData16(), len); + return; + } auto internal = valueInternal().impl(); @@ -967,8 +1223,14 @@ inline GCOwnedDataScope JSString::tryGetValue(bool allocationAllo // Pass nullptr for the JSGlobalObject so that resolveRope does not throw in the event of an OOM error. return { this, static_cast(this)->resolveRope(nullptr) }; } +#if USE(BUN_JSC_ADDITIONS) + if (isInline()) { + BUN_INLINE_COUNT(g_bunInlineResolvedTryGetValue); + return { this, resolveInline(nullptr) }; + } +#endif } else - RELEASE_ASSERT(!isRope()); + RELEASE_ASSERT(!(m_fiber & notStringImplMask)); return { this, valueInternal() }; } @@ -1025,10 +1287,69 @@ inline JSString* jsString(VM& vm, StringView s) if (auto c = s.codeUnitAt(0); c <= maxSingleCharacterString) return vm.smallStrings.singleCharacterString(c); } +#if USE(BUN_JSC_ADDITIONS) + if (s.is8Bit()) { + if (static_cast(size) <= JSString::maxInlineLength8) + return JSString::createInline8(vm, s.span8()); + if (static_cast(size) <= JSString::maxBigInlineLength8) + return JSBigInlineString::create8(vm, s.span8()); + } else { + if (static_cast(size) <= JSString::maxInlineLength16) + return JSString::createInline16(vm, s.span16()); + if (static_cast(size) <= JSString::maxBigInlineLength16) + return JSBigInlineString::create16(vm, s.span16()); + } +#endif auto impl = s.is8Bit() ? StringImpl::create(s.span8()) : StringImpl::create(s.span16()); return JSString::create(vm, WTF::move(impl)); } +#if USE(BUN_JSC_ADDITIONS) +ALWAYS_INLINE JSString* JSString::createInline8(VM& vm, std::span chars) +{ + uintptr_t fiber = encodeInline8(chars); + if (JSString* cached = vm.inlineStringCache.lookup(fiber)) { + BUN_INLINE_COUNT(g_bunInlineCacheHit); + return cached; + } + JSString* s = createInlineFromFiber(vm, fiber); + vm.inlineStringCache.insert(fiber, s); + return s; +} + +ALWAYS_INLINE JSString* JSString::createInline16(VM& vm, std::span chars) +{ + uintptr_t fiber = encodeInline16(chars); + if (JSString* cached = vm.inlineStringCache.lookup(fiber)) + return cached; + JSString* s = createInlineFromFiber(vm, fiber); + vm.inlineStringCache.insert(fiber, s); + return s; +} + +ALWAYS_INLINE JSString* JSString::createInline8(VM& vm, GCDeferralContext* deferralContext, std::span chars) +{ + uintptr_t fiber = encodeInline8(chars); + if (JSString* cached = vm.inlineStringCache.lookup(fiber)) { + BUN_INLINE_COUNT(g_bunInlineCacheHit); + return cached; + } + JSString* s = createInlineFromFiber(vm, deferralContext, fiber); + vm.inlineStringCache.insert(fiber, s); + return s; +} + +ALWAYS_INLINE JSString* JSString::createInline16(VM& vm, GCDeferralContext* deferralContext, std::span chars) +{ + uintptr_t fiber = encodeInline16(chars); + if (JSString* cached = vm.inlineStringCache.lookup(fiber)) + return cached; + JSString* s = createInlineFromFiber(vm, deferralContext, fiber); + vm.inlineStringCache.insert(fiber, s); + return s; +} +#endif + ALWAYS_INLINE JSString* jsString(VM& vm, RefPtr&& s) { return jsString(vm, String { WTF::move(s) }); @@ -1161,6 +1482,65 @@ inline JSString* jsOwnedString(VM& vm, const String& s) return JSString::createHasOtherOwner(vm, *s.impl()); } +#if USE(BUN_JSC_ADDITIONS) +// audit-string-materialize: central helper so Object.entries' pre-existing +// inline-cell sites share one fiber→JSString path. Inline-fiber JSString cells +// are NOT DFG-safe (StringCharAt/substring/length inline m_fiber→m_length with +// no inline-cell guard), so keys that flow into user JS stay atom-backed: +// - ObjectConstructor.cpp objectConstructorEntries → REWRITE (here) +// - ObjectConstructor.cpp getPropertyKeys → jsOwnedAtomBackedString (user JS) +// - JSPropertyNameEnumerator.cpp:finishCreation → KEEP (for-in→user JS) +// - Lookup.h reifyStaticProperty → publicName() → KEEP (cold host-fn) +// - LiteralParser.cpp → KEEP (span→Identifier only) +// - PropertyName::publicName() → KEEP (already fiber-aware) +ALWAYS_INLINE JSString* jsStringFromFiberOrImpl(VM& vm, UniquedStringImpl* uid) +{ + uintptr_t bits = reinterpret_cast(uid); + if (isInlinePropertyKey(bits)) { + if (JSString* cached = vm.inlineStringCache.lookup(bits)) + return cached; + JSString* s = JSString::createInlineFromFiber(vm, bits); + vm.inlineStringCache.insert(bits, s); + return s; + } + ASSERT(uid); + unsigned length = uid->length(); + if (!length) + return vm.smallStrings.emptyString(); + if (length == 1) { + char16_t c = uid->is8Bit() ? uid->span8()[0] : uid->span16()[0]; + if (c <= maxSingleCharacterString) + return vm.smallStrings.singleCharacterString(c); + } + return JSString::createHasOtherOwner(vm, *uid); +} + +ALWAYS_INLINE JSString* jsStringFromFiberOrImpl(VM& vm, const Identifier& identifier) +{ + return jsStringFromFiberOrImpl(vm, identifier.impl()); +} + +// DFG-safe (always atom-backed) variant of jsStringFromFiberOrImpl for keys that +// flow into user JS: fiber words materialize via string(); real impls skip the +// by-value string() AtomString temporary (splay/raytrace protected-win regressor). +ALWAYS_INLINE JSString* jsOwnedAtomBackedString(VM& vm, const Identifier& identifier) +{ + UniquedStringImpl* uid = identifier.impl(); + if (isInlinePropertyKey(uid)) [[unlikely]] + return jsOwnedString(vm, identifier.string()); // materialize atom; DFG-safe + ASSERT(uid); + unsigned length = uid->length(); + if (!length) + return vm.smallStrings.emptyString(); + if (length == 1) { + char16_t c = uid->is8Bit() ? uid->span8()[0] : uid->span16()[0]; + if (c <= maxSingleCharacterString) + return vm.smallStrings.singleCharacterString(c); + } + return JSString::createHasOtherOwner(vm, *uid); +} +#endif + ALWAYS_INLINE JSString* jsStringWithCache(VM& vm, const String& s) { unsigned length = s.length(); @@ -1244,12 +1624,31 @@ ALWAYS_INLINE GCOwnedDataScope JSString::view(JSGlobalObject* global { if (isRope()) return static_cast(*this).view(globalObject); +#if USE(BUN_JSC_ADDITIONS) + // The returned StringView points at &m_fiber+1. This is safe for the + // lifetime of the GCOwnedDataScope: the cell is kept alive, and nothing + // else resolves the inline string in place while the scope is live. + // Callers that outlive the scope or call value()/tryGetValue() on the same + // string afterwards must resolve first (see JSStringJoiner). + uintptr_t fiber = fiberConcurrently(); + if (isInlineFiber(fiber)) { + unsigned len = inlineLengthFromFiber(fiber); + if (fiber & JSRopeString::is8BitInPointer) + return { this, StringView { std::span { inlineData8(), len } } }; + return { this, StringView { std::span { inlineData16(), len } } }; + } +#endif return { this, valueInternal() }; } inline bool JSString::isSubstring() const { +#if USE(BUN_JSC_ADDITIONS) + // isSubstringInPointer == isInlineInPointer; a substring rope has both low bits set. + return (fiberConcurrently() & notStringImplMask) == (isRopeInPointer | JSRopeString::isSubstringInPointer); +#else return fiberConcurrently() & JSRopeString::isSubstringInPointer; +#endif } } // namespace JSC diff --git a/Source/JavaScriptCore/runtime/JSStringBuilder.h b/Source/JavaScriptCore/runtime/JSStringBuilder.h new file mode 100644 index 0000000000000..42159f7454c98 --- /dev/null +++ b/Source/JavaScriptCore/runtime/JSStringBuilder.h @@ -0,0 +1,200 @@ +/* + * Copyright (C) 2024 Apple Inc. All rights reserved. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public License + * along with this library; see the file COPYING.LIB. If not, write to + * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + +#pragma once + +#if USE(BUN_JSC_ADDITIONS) + +#include "JSString.h" +#include +#include +#include + +namespace JSC { + +// A StringBuilder drop-in for sites that end with jsString(vm, builder.toString()). +// Short all-Latin-1 results (≤ maxBigInlineLength8) accumulate in a stack buffer +// and emit an inline JSString cell directly from toJS(), touching no malloc. +class JSStringBuilder { + WTF_MAKE_NONCOPYABLE(JSStringBuilder); + WTF_FORBID_HEAP_ALLOCATION; + +public: + static constexpr unsigned inlineCapacity = JSString::maxBigInlineLength8; + + JSStringBuilder() = default; + explicit JSStringBuilder(OverflowPolicy policy) + : m_builder(policy) + { + } + + ALWAYS_INLINE void append(Latin1Character c) + { + if (m_spilled) { + m_builder.append(c); + return; + } + if (m_inlineLength < inlineCapacity) [[likely]] { + m_inline[m_inlineLength++] = c; + return; + } + spill(); + m_builder.append(c); + } + + ALWAYS_INLINE void append(char c) { append(byteCast(c)); } + + ALWAYS_INLINE void append(char16_t c) + { + if (m_spilled) { + m_builder.append(c); + return; + } + if (isLatin1(c)) { + append(static_cast(c)); + return; + } + spill(); + m_builder.append(c); + } + + void append(std::span chars) + { + if (m_spilled) { + m_builder.append(chars); + return; + } + if (m_inlineLength + chars.size() <= inlineCapacity) { + if (!chars.empty()) + memcpy(m_inline + m_inlineLength, chars.data(), chars.size()); + m_inlineLength += chars.size(); + return; + } + spill(); + m_builder.append(chars); + } + + void append(std::span chars) + { + if (!m_spilled) + spill(); + m_builder.append(chars); + } + + ALWAYS_INLINE void append(StringView s) + { + if (s.is8Bit()) + append(s.span8()); + else + append(s.span16()); + } + + ALWAYS_INLINE void append(const String& s) { append(StringView { s }); } + ALWAYS_INLINE void append(const StringBuilder& other) { append(StringView { other }); } + ALWAYS_INLINE void append(ASCIILiteral s) { append(s.span8()); } + + // Variadic: hex(), numbers, makeString()-style args. + template + void append(const StringTypes&... strings) + { + appendFromAdapters(WTF::StringTypeAdapter(strings)...); + } + + void reserveCapacity(unsigned capacity) + { + if (capacity <= inlineCapacity) + return; + if (!m_spilled) + spill(); + m_builder.reserveCapacity(capacity); + } + + bool hasOverflowed() const { return m_spilled && m_builder.hasOverflowed(); } + bool isEmpty() const { return !length(); } + unsigned length() const { return m_spilled ? m_builder.length() : m_inlineLength; } + bool is8Bit() const { return !m_spilled || m_builder.is8Bit(); } + + JSString* toJS(VM& vm) + { + if (!m_spilled) { + if (!m_inlineLength) + return vm.smallStrings.emptyString(); + if (m_inlineLength == 1) + return vm.smallStrings.singleCharacterString(m_inline[0]); + if (m_inlineLength <= JSString::maxInlineLength8) + return JSString::createInline8(vm, { m_inline, m_inlineLength }); + return JSBigInlineString::create8(vm, { m_inline, m_inlineLength }); + } + // Spilled: still check for inline-eligible short results (e.g. a char16_t + // forced spill). Long results reuse the builder's StringImpl via toString(). + unsigned len = m_builder.length(); + if (len <= JSString::maxBigInlineLength8 && (m_builder.is8Bit() || len <= JSString::maxBigInlineLength16)) + return jsString(vm, StringView { m_builder }); + return jsString(vm, m_builder.toString()); + } + + // Escape hatch for callers that also need the WTF::String. + String toString() + { + if (!m_spilled) + spill(); + return m_builder.toString(); + } + +private: + template + ALWAYS_INLINE void appendFromAdapters(const Adapters&... adapters) + { + if (m_spilled) { + m_builder.appendFromAdapters(adapters...); + return; + } + if constexpr (!(... || WTF::stringBuilderSlowPathRequiredForAdapter)) { + if (WTF::are8Bit(adapters...)) { + auto required = WTF::saturatingSum(static_cast(m_inlineLength), adapters.length()...); + if (required <= inlineCapacity) { + std::span dest { m_inline + m_inlineLength, static_cast(required - m_inlineLength) }; + WTF::stringTypeAdapterAccumulator(dest, adapters...); + m_inlineLength = static_cast(required); + return; + } + } + } + spill(); + m_builder.appendFromAdapters(adapters...); + } + + ALWAYS_INLINE void spill() + { + ASSERT(!m_spilled); + m_spilled = true; + if (m_inlineLength) + m_builder.append(std::span { m_inline, m_inlineLength }); + } + + StringBuilder m_builder; + Latin1Character m_inline[inlineCapacity]; + uint8_t m_inlineLength { 0 }; + bool m_spilled { false }; +}; + +} // namespace JSC + +#endif // USE(BUN_JSC_ADDITIONS) diff --git a/Source/JavaScriptCore/runtime/JSStringInlines.h b/Source/JavaScriptCore/runtime/JSStringInlines.h index 097f109aa7173..913ea31d5076c 100644 --- a/Source/JavaScriptCore/runtime/JSStringInlines.h +++ b/Source/JavaScriptCore/runtime/JSStringInlines.h @@ -37,9 +37,35 @@ namespace JSC { +#if USE(BUN_JSC_ADDITIONS) +// StringView over a non-rope JSString's characters that is safe for inline +// small strings (m_fiber holds packed chars, not a StringImpl*). +ALWAYS_INLINE static StringView nonRopeStringView(const JSString* string) +{ + ASSERT(!string->isRope()); + if (string->isInline()) { + unsigned len = string->length(); + if (string->is8Bit()) + return StringView { std::span { string->inlineData8(), len } }; + return StringView { std::span { string->inlineData16(), len } }; + } + return StringView { string->getValueImpl() }; +} +#else +ALWAYS_INLINE static StringView nonRopeStringView(const JSString* string) +{ + ASSERT(!string->isRope()); + return StringView { string->getValueImpl() }; +} +#endif + ALWAYS_INLINE void JSString::destroy(JSCell* cell) { auto* string = static_cast(cell); +#if USE(BUN_JSC_ADDITIONS) + if (string->m_fiber & notStringImplMask) + return; +#endif string->valueInternal().~String(); } @@ -53,7 +79,7 @@ ALWAYS_INLINE void JSRopeString::destroy(JSCell* cell) bool JSString::equal(JSGlobalObject* globalObject, JSString* other) const { - if (isRope() || other->isRope()) + if ((m_fiber | other->m_fiber) & notStringImplMask) return equalSlowCase(globalObject, other); return WTF::equal(*valueInternal().impl(), *other->valueInternal().impl()); } @@ -206,7 +232,7 @@ std::optional JSString::tryFindOneChar(JSGlobalObject*, char16_t charact if (!fiber->isRope()) { unsigned localStart = startPosition > offset ? startPosition - offset : 0; - size_t result = StringView(fiber->valueInternal()).find(character, localStart); + size_t result = nonRopeStringView(fiber).find(character, localStart); if (result != WTF::notFound) return offset + result; } else if (fiber->isSubstring()) { @@ -281,7 +307,7 @@ std::optional JSString::tryFindLastOneChar(JSGlobalObject*, char16_t cha JSString* fiber = fibers[i]; if (!fiber->isRope()) { unsigned localStart = startPosition >= offset + fiberLength ? fiberLength - 1 : startPosition - offset; - size_t result = StringView(fiber->valueInternal()).reverseFind(character, localStart); + size_t result = nonRopeStringView(fiber).reverseFind(character, localStart); if (result != WTF::notFound) return offset + result; } else if (fiber->isSubstring()) { @@ -321,7 +347,7 @@ ALWAYS_INLINE std::optional JSString::tryGetCharAt(JSGlobalObject*, un unsigned localIndex = index - offset; if (!fiber->isRope()) - return StringView(fiber->valueInternal())[localIndex]; + return nonRopeStringView(fiber)[localIndex]; if (fiber->isSubstring()) { const JSRopeString* substringFiber = static_cast(fiber); return StringView(substringFiber->substringBase()->valueInternal())[substringFiber->substringOffset() + localIndex]; @@ -439,7 +465,7 @@ WTF_ALLOW_UNSAFE_BUFFER_USAGE_BEGIN continue; } - StringView view = *currentFiber->valueInternal().impl(); + StringView view = nonRopeStringView(currentFiber); position -= view.length(); view.getCharacters(unsafeMakeSpan(position, end - position)); } while (!workQueue.isEmpty()); @@ -475,7 +501,7 @@ inline void JSRopeString::resolveToBuffer(JSString* fiber0, JSString* fiber1, JS } skip(buffer, rope0Length); } else { - StringView view0 = fiber0->valueInternal().impl(); + StringView view0 = nonRopeStringView(fiber0); view0.getCharacters(buffer); skip(buffer, view0.length()); } @@ -559,7 +585,7 @@ inline void JSRopeString::resolveToBuffer(JSString* fiber0, JSString* fiber1, JS auto* rope0 = static_cast(fiber0); auto rope0Length = rope0->length(); { - StringView view1 = fiber1->valueInternal().impl(); + StringView view1 = nonRopeStringView(fiber1); view1.getCharacters(buffer.subspan(rope0Length)); } if (rope0->isSubstring()) { @@ -576,7 +602,7 @@ inline void JSRopeString::resolveToBuffer(JSString* fiber0, JSString* fiber1, JS auto* rope1 = static_cast(fiber1); auto rope1Length = rope1->length(); { - StringView view0 = fiber0->valueInternal().impl(); + StringView view0 = nonRopeStringView(fiber0); view0.getCharacters(buffer); skip(buffer, view0.length()); } @@ -589,16 +615,16 @@ inline void JSRopeString::resolveToBuffer(JSString* fiber0, JSString* fiber1, JS MUST_TAIL_CALL return resolveToBuffer(rope1->fiber0(), rope1->fiber1(), rope1->fiber2(), buffer.first(rope1Length), stackLimit); } - StringView view0 = fiber0->valueInternal().impl(); + StringView view0 = nonRopeStringView(fiber0); view0.getCharacters(buffer); - StringView view1 = fiber1->valueInternal().impl(); + StringView view1 = nonRopeStringView(fiber1); view1.getCharacters(buffer.subspan(view0.length())); return; } // 1 fiber. if (!fiber0->isRope()) { - StringView view0 = fiber0->valueInternal().impl(); + StringView view0 = nonRopeStringView(fiber0); view0.getCharacters(buffer); return; } @@ -629,6 +655,10 @@ inline JSString* jsAtomString(JSGlobalObject* globalObject, VM& vm, JSString* st } if (!string->isRope()) { +#if USE(BUN_JSC_ADDITIONS) + if (string->isInline()) + string->resolveInline(globalObject); +#endif auto createFromNonRope = [&](VM& vm, auto&) { AtomString atom(string->valueInternal()); if (!string->valueInternal().impl()->isAtom()) @@ -719,7 +749,7 @@ inline JSString* jsAtomString(JSGlobalObject* globalObject, VM& vm, JSString* s1 return JSRopeString::resolveToBufferSlow(fiber0, fiber1, nullptr, buffer, stackLimit); auto* rope0 = static_cast(fiber0); - StringView view1 = fiber1->valueInternal().impl(); + StringView view1 = nonRopeStringView(fiber1); view1.getCharacters(buffer.subspan(rope0->length())); if (rope0->isSubstring()) { StringView view0 = *rope0->substringBase()->valueInternal().impl(); @@ -731,7 +761,7 @@ inline JSString* jsAtomString(JSGlobalObject* globalObject, VM& vm, JSString* s1 } if (fiber1->isRope()) { - StringView view0 = fiber0->valueInternal().impl(); + StringView view0 = nonRopeStringView(fiber0); view0.getCharacters(buffer); auto* rope1 = static_cast(fiber1); if (rope1->isSubstring()) { @@ -743,9 +773,9 @@ inline JSString* jsAtomString(JSGlobalObject* globalObject, VM& vm, JSString* s1 return JSRopeString::resolveToBuffer(rope1->fiber0(), rope1->fiber1(), rope1->fiber2(), buffer.subspan(view0.length(), rope1->length()), stackLimit); } - StringView view0 = fiber0->valueInternal().impl(); + StringView view0 = nonRopeStringView(fiber0); view0.getCharacters(buffer); - StringView view1 = fiber1->valueInternal().impl(); + StringView view1 = nonRopeStringView(fiber1); view1.getCharacters(buffer.subspan(view0.length())); }; @@ -800,11 +830,11 @@ inline JSString* jsAtomString(JSGlobalObject* globalObject, VM& vm, JSString* s1 if (fiber0->isRope() || fiber1->isRope() || fiber2->isRope()) return JSRopeString::resolveToBufferSlow(fiber0, fiber1, fiber2, buffer, std::bit_cast(vm.softStackLimit())); - StringView view0 = fiber0->valueInternal().impl(); + StringView view0 = nonRopeStringView(fiber0); view0.getCharacters(buffer); - StringView view1 = fiber1->valueInternal().impl(); + StringView view1 = nonRopeStringView(fiber1); view1.getCharacters(buffer.subspan(view0.length())); - StringView view2 = fiber2->valueInternal().impl(); + StringView view2 = nonRopeStringView(fiber2); view2.getCharacters(buffer.subspan(view0.length() + view1.length())); }; @@ -837,6 +867,29 @@ inline JSString* jsSubstringOfResolved(VM& vm, GCDeferralContext* deferralContex } ASSERT(!s->isRope()); +#if USE(BUN_JSC_ADDITIONS) + // Substring of an inline input: copy the bytes into a fresh inline cell + // without resolving the input (no StringImpl allocation). Must honour + // deferralContext — callers like createRegExpMatchesArray have an + // uninitialised-butterfly array live on the stack. + if (s->isInline()) { + uintptr_t fiber = s->fiberConcurrently(); + if (fiber & JSRopeString::is8BitInPointer) { + auto span = std::span { s->inlineData8(), JSString::inlineLengthFromFiber(fiber) }.subspan(offset, length); + if (length == 1) + return vm.smallStrings.singleCharacterString(span[0]); + if (length <= JSString::maxInlineLength8) + return JSString::createInline8(vm, deferralContext, span); + return JSBigInlineString::create8(vm, deferralContext, span); + } + auto span = std::span { s->inlineData16(), JSString::inlineLengthFromFiber(fiber) }.subspan(offset, length); + if (length == 1 && span[0] <= maxSingleCharacterString) + return vm.smallStrings.singleCharacterString(span[0]); + if (length <= JSString::maxInlineLength16) + return JSString::createInline16(vm, deferralContext, span); + return JSBigInlineString::create16(vm, deferralContext, span); + } +#endif auto& base = s->valueInternal(); if (!offset && length == base.length()) return s; @@ -857,6 +910,20 @@ inline JSString* jsSubstringOfResolved(VM& vm, GCDeferralContext* deferralContex return vm.keyAtomStringCache.make(vm, buffer, createFromSubstring); } } +#if USE(BUN_JSC_ADDITIONS) + // Short substrings: 16/24-byte inline cell instead of a 32-byte substring rope. + if (base.is8Bit()) { + if (length <= JSString::maxInlineLength8) + return JSString::createInline8(vm, deferralContext, base.span8().subspan(offset, length)); + if (length <= JSString::maxBigInlineLength8) + return JSBigInlineString::create8(vm, deferralContext, base.span8().subspan(offset, length)); + } else { + if (length <= JSString::maxInlineLength16) + return JSString::createInline16(vm, deferralContext, base.span16().subspan(offset, length)); + if (length <= JSString::maxBigInlineLength16) + return JSBigInlineString::create16(vm, deferralContext, base.span16().subspan(offset, length)); + } +#endif return JSRopeString::createSubstringOfResolved(vm, deferralContext, s, offset, length, base.is8Bit()); } @@ -875,6 +942,17 @@ void JSString::resolveToBuffer(std::span destination) uint8_t* stackLimit = std::bit_cast(vm().softStackLimit()); return JSRopeString::resolveToBuffer(rope->fiber0(), rope->fiber1(), rope->fiber2(), destination, stackLimit); } +#if USE(BUN_JSC_ADDITIONS) + uintptr_t fiber = fiberConcurrently(); + if (isInlineFiber(fiber)) { + unsigned len = inlineLengthFromFiber(fiber); + if (fiber & JSRopeString::is8BitInPointer) + StringView(std::span { inlineData8(), len }).getCharacters(destination); + else + StringView(std::span { inlineData16(), len }).getCharacters(destination); + return; + } +#endif StringView(valueInternal().impl()).getCharacters(destination); } diff --git a/Source/JavaScriptCore/runtime/JSStringJoiner.h b/Source/JavaScriptCore/runtime/JSStringJoiner.h index 2a2e4288e6ea1..45cc3436f906f 100644 --- a/Source/JavaScriptCore/runtime/JSStringJoiner.h +++ b/Source/JavaScriptCore/runtime/JSStringJoiner.h @@ -148,6 +148,16 @@ ALWAYS_INLINE bool JSStringJoiner::appendWithoutSideEffects(JSGlobalObject* glob // FIXME: Support JSBigInt in side-effect-free append. // https://bugs.webkit.org/show_bug.cgi?id=211173 if (JSString* jsString = dynamicDowncast(value)) { +#if USE(BUN_JSC_ADDITIONS) + // view() for an inline cell points at &m_fiber+1, which tryGetValue() + // below would overwrite. Resolve directly to an atom so repeated + // short contents share a single AtomStringImpl instead of allocating + // a throwaway StringImpl per element. + if (jsString->isInline()) { + BUN_INLINE_COUNT(g_bunInlineResolvedJoiner); + jsString->resolveInlineToAtomString(globalObject); + } +#endif auto view = jsString->view(globalObject); RETURN_IF_EXCEPTION(scope, false); // Since getting the view didn't OOM, we know that the underlying String exists and isn't @@ -200,6 +210,10 @@ ALWAYS_INLINE bool JSStringJoiner::append(JSGlobalObject* globalObject, JSValue ASSERT(!value.isString()); JSString* jsString = value.asCell()->toStringInline(globalObject); RETURN_IF_EXCEPTION(scope, false); +#if USE(BUN_JSC_ADDITIONS) + if (jsString->isInline()) + jsString->resolveInlineToAtomString(globalObject); +#endif auto view = jsString->view(globalObject); RETURN_IF_EXCEPTION(scope, false); scope.release(); diff --git a/Source/JavaScriptCore/runtime/JSSymbolTableObject.cpp b/Source/JavaScriptCore/runtime/JSSymbolTableObject.cpp index cb151b9d79a81..8505e41eeeb11 100644 --- a/Source/JavaScriptCore/runtime/JSSymbolTableObject.cpp +++ b/Source/JavaScriptCore/runtime/JSSymbolTableObject.cpp @@ -66,8 +66,13 @@ void JSSymbolTableObject::getOwnSpecialPropertyNames(JSObject* object, JSGlobalO SymbolTable::Map::iterator end = symbolTable->end(locker); for (SymbolTable::Map::iterator it = symbolTable->begin(locker); it != end; ++it) { if (mode == DontEnumPropertiesMode::Include || !it->value.isDontEnum()) { +#if USE(BUN_JSC_ADDITIONS) + if (!propertyNames.includeSymbolProperties() && uidIsSymbol(it->key.get())) + continue; +#else if (!propertyNames.includeSymbolProperties() && it->key->isSymbol()) continue; +#endif if (propertyNames.privateSymbolMode() == PrivateSymbolMode::Exclude && symbolTable->hasPrivateName(it->key)) continue; propertyNames.add(Identifier::fromUid(vm, it->key.get())); diff --git a/Source/JavaScriptCore/runtime/LiteralParser.cpp b/Source/JavaScriptCore/runtime/LiteralParser.cpp index 8ead0c30b1c9f..231f09d15df1d 100644 --- a/Source/JavaScriptCore/runtime/LiteralParser.cpp +++ b/Source/JavaScriptCore/runtime/LiteralParser.cpp @@ -149,6 +149,40 @@ bool LiteralParser::tryJSONPParse(Vector& resu template ALWAYS_INLINE bool LiteralParser::equalIdentifier(UniquedStringImpl* rep, typename Lexer::LiteralParserTokenPtr token) { +#if USE(BUN_JSC_ADDITIONS) + // Structure::transitionPropertyName() may hand us a fiber-word-tagged + // UniquedStringImpl* (2..5-char Latin-1 keys). Decode and compare spans + // instead of dereferencing. + if (isInlinePropertyKey(rep)) { + uintptr_t word = reinterpret_cast(rep); + unsigned len = inlinePropertyKeyLength(word); + const uint8_t* bytes = reinterpret_cast(&word); + auto equalSpan = [&](auto tokenSpan) ALWAYS_INLINE_LAMBDA -> bool { + if (tokenSpan.size() != len) + return false; + if (inlinePropertyKeyIs8Bit(word)) { + const Latin1Character* chars = bytes + 1; + for (unsigned i = 0; i < len; ++i) { + if (static_cast(chars[i]) != static_cast(tokenSpan[i])) + return false; + } + } else { + const char16_t* chars = reinterpret_cast(bytes + 2); + for (unsigned i = 0; i < len; ++i) { + if (chars[i] != static_cast(tokenSpan[i])) + return false; + } + } + return true; + }; + if (token->type == TokIdentifier) + return equalSpan(token->identifier()); + ASSERT(token->type == TokString); + if (token->stringIs8Bit) + return equalSpan(token->string8()); + return equalSpan(token->string16()); + } +#endif // In the literal parser, we don't want to follow property addition transitions if the property name is a symbol. if (rep->isSymbol()) return false; diff --git a/Source/JavaScriptCore/runtime/Lookup.h b/Source/JavaScriptCore/runtime/Lookup.h index a7134ccf7e0b9..5217a5d8f7657 100644 --- a/Source/JavaScriptCore/runtime/Lookup.h +++ b/Source/JavaScriptCore/runtime/Lookup.h @@ -347,11 +347,35 @@ struct HashTable { if (!uid) return nullptr; +#if USE(BUN_JSC_ADDITIONS) + // Static tables are indexed by the build-time content hash (create_hash_table's + // perl rapidhash), NOT uidHash() which hashes the pointer bits. Probe with the + // StringImpl content hash / inlinePropertyKeyHash so we match the generated index. + uintptr_t uidWord = reinterpret_cast(uid); + int indexEntry = (isInlinePropertyKey(uidWord) ? inlinePropertyKeyHash(uidWord) : uid->hash()) & indexMask; +#else int indexEntry = IdentifierRepHash::hash(uid) & indexMask; +#endif int valueIndex = index[indexEntry].value; if (valueIndex == -1) return nullptr; +#if USE(BUN_JSC_ADDITIONS) + if (isInlinePropertyKey(uidWord)) [[unlikely]] { + auto keySpan = inlinePropertyKeySpan8(uidWord); + while (true) { + auto& tableKey = values[valueIndex].m_key; + if (!tableKey.isNull() && tableKey.length() == keySpan.size() + && !memcmp(tableKey.characters(), keySpan.data(), keySpan.size())) + return &values[valueIndex]; + indexEntry = index[indexEntry].next; + if (indexEntry == -1) + return nullptr; + valueIndex = index[indexEntry].value; + ASSERT(valueIndex != -1); + } + } +#endif while (true) { if (!values[valueIndex].m_key.isNull() && WTF::equal(uid, values[valueIndex].m_key)) return &values[valueIndex]; diff --git a/Source/JavaScriptCore/runtime/MegamorphicCache.h b/Source/JavaScriptCore/runtime/MegamorphicCache.h index 06d63b1cbd4c3..df46870be5e39 100644 --- a/Source/JavaScriptCore/runtime/MegamorphicCache.h +++ b/Source/JavaScriptCore/runtime/MegamorphicCache.h @@ -28,6 +28,10 @@ #include "Structure.h" #include +#if USE(BUN_JSC_ADDITIONS) +#include "InlinePropertyKey.h" +#endif + namespace JSC { DECLARE_ALLOCATOR_WITH_HEAP_IDENTIFIER(MegamorphicCache); @@ -85,7 +89,11 @@ class MegamorphicCache { m_holder = (ownProperty) ? JSCell::seenMultipleCalleeObjects() : holder; } +#if USE(BUN_JSC_ADDITIONS) + FiberAwareRefPtr m_uid; +#else RefPtr m_uid; +#endif StructureID m_structureID { }; uint16_t m_epoch { invalidEpoch }; uint16_t m_offset { 0 }; @@ -110,7 +118,11 @@ class MegamorphicCache { m_reallocating = reallocating; } +#if USE(BUN_JSC_ADDITIONS) + FiberAwareRefPtr m_uid; +#else RefPtr m_uid; +#endif StructureID m_oldStructureID { }; StructureID m_newStructureID { }; uint16_t m_epoch { invalidEpoch }; @@ -132,7 +144,11 @@ class MegamorphicCache { m_result = !!result; } +#if USE(BUN_JSC_ADDITIONS) + FiberAwareRefPtr m_uid; +#else RefPtr m_uid; +#endif StructureID m_structureID { }; uint16_t m_epoch { invalidEpoch }; uint16_t m_result { false }; @@ -170,7 +186,11 @@ class MegamorphicCache { ALWAYS_INLINE static uint32_t primaryHash(StructureID structureID, UniquedStringImpl* uid) { uint32_t sid = std::bit_cast(structureID); +#if USE(BUN_JSC_ADDITIONS) + return ((sid >> structureIDHashShift1) ^ (sid >> structureIDHashShift2)) + uidHash(uid); +#else return ((sid >> structureIDHashShift1) ^ (sid >> structureIDHashShift2)) + uid->hash(); +#endif } ALWAYS_INLINE static uint32_t secondaryHash(StructureID structureID, UniquedStringImpl* uid) @@ -182,7 +202,11 @@ class MegamorphicCache { ALWAYS_INLINE static uint32_t storeCachePrimaryHash(StructureID structureID, UniquedStringImpl* uid) { uint32_t sid = std::bit_cast(structureID); +#if USE(BUN_JSC_ADDITIONS) + return ((sid >> structureIDHashShift1) ^ (sid >> structureIDHashShift4)) + uidHash(uid); +#else return ((sid >> structureIDHashShift1) ^ (sid >> structureIDHashShift4)) + uid->hash(); +#endif } ALWAYS_INLINE static uint32_t storeCacheSecondaryHash(StructureID structureID, UniquedStringImpl* uid) @@ -194,7 +218,11 @@ class MegamorphicCache { ALWAYS_INLINE static uint32_t hasCachePrimaryHash(StructureID structureID, UniquedStringImpl* uid) { uint32_t sid = std::bit_cast(structureID); +#if USE(BUN_JSC_ADDITIONS) + return ((sid >> structureIDHashShift1) ^ (sid >> structureIDHashShift6)) + uidHash(uid); +#else return ((sid >> structureIDHashShift1) ^ (sid >> structureIDHashShift6)) + uid->hash(); +#endif } ALWAYS_INLINE static uint32_t hasCacheSecondaryHash(StructureID structureID, UniquedStringImpl* uid) diff --git a/Source/JavaScriptCore/runtime/ObjectConstructor.cpp b/Source/JavaScriptCore/runtime/ObjectConstructor.cpp index 676609f45f725..3526f2bc2e565 100644 --- a/Source/JavaScriptCore/runtime/ObjectConstructor.cpp +++ b/Source/JavaScriptCore/runtime/ObjectConstructor.cpp @@ -421,7 +421,12 @@ JSC_DEFINE_HOST_FUNCTION(objectConstructorEntries, (JSGlobalObject* globalObject } { +#if USE(BUN_JSC_ADDITIONS) + // entry.key() may be a fiber-word-tagged UniquedStringImpl*; use fiber-aware ref/deref. + Vector properties; +#else Vector, 8> properties; +#endif MarkedArgumentBuffer values; bool canUseFastPath = false; if (!target->canHaveExistingOwnIndexedProperties() && !target->hasNonReifiedStaticProperties()) { @@ -432,8 +437,13 @@ JSC_DEFINE_HOST_FUNCTION(objectConstructorEntries, (JSGlobalObject* globalObject if (entry.attributes() & PropertyAttribute::DontEnum) return true; +#if USE(BUN_JSC_ADDITIONS) + if (uidIsSymbol(entry.key())) + return true; +#else if (entry.key()->isSymbol()) return true; +#endif properties.append(entry.key()); values.appendWithCrashOnOverflow(target->getDirect(entry.offset())); @@ -469,7 +479,11 @@ JSC_DEFINE_HOST_FUNCTION(objectConstructorEntries, (JSGlobalObject* globalObject auto* newButterfly = JSCellButterfly::create(vm, CopyOnWriteArrayWithContiguous, numProperties); for (size_t i = 0; i < numProperties; i++) { const auto& identifier = properties[i]; +#if USE(BUN_JSC_ADDITIONS) + newButterfly->setIndex(vm, i, jsStringFromFiberOrImpl(vm, identifier.get())); +#else newButterfly->setIndex(vm, i, jsOwnedString(vm, identifier.get())); +#endif } targetStructure->setCachedPropertyNames(vm, CachedPropertyNamesKind::EnumerableStrings, newButterfly); @@ -482,12 +496,26 @@ JSC_DEFINE_HOST_FUNCTION(objectConstructorEntries, (JSGlobalObject* globalObject JSString* key = nullptr; if (cachedButterfly) { auto* cachedKey = asString(cachedButterfly->get(i)); +#if USE(BUN_JSC_ADDITIONS) + UniquedStringImpl* uid = properties[i].get(); + bool hit = isInlinePropertyKey(uid) + ? (cachedKey->isInline() && cachedKey->inlineFiberWord() == inlinePropertyKeyWord(uid)) + : (cachedKey->tryGetValueImpl() == uid); + if (hit) + key = cachedKey; +#else if (cachedKey->tryGetValueImpl() == properties[i].get()) key = cachedKey; +#endif } - if (!key) + if (!key) { +#if USE(BUN_JSC_ADDITIONS) + key = jsStringFromFiberOrImpl(vm, properties[i].get()); +#else key = jsOwnedString(vm, properties[i].get()); +#endif + } JSArray* entry = nullptr; { @@ -533,7 +561,11 @@ JSC_DEFINE_HOST_FUNCTION(objectConstructorEntries, (JSGlobalObject* globalObject value = target->get(globalObject, propertyName); RETURN_IF_EXCEPTION(scope, void()); +#if USE(BUN_JSC_ADDITIONS) + JSString* key = jsStringFromFiberOrImpl(vm, propertyName.uid()); +#else JSString* key = jsOwnedString(vm, propertyName.uid()); +#endif JSArray* entry = nullptr; { ObjectInitializationScope initializationScope(vm); @@ -580,8 +612,13 @@ JSValue objectValues(VM& vm, JSGlobalObject* globalObject, JSValue targetValue) if (entry.attributes() & PropertyAttribute::DontEnum) return true; +#if USE(BUN_JSC_ADDITIONS) + if (uidIsSymbol(entry.key())) + return true; +#else if (entry.key()->isSymbol()) return true; +#endif namedPropertyValues.appendWithCrashOnOverflow(target->getDirect(entry.offset())); return true; @@ -885,7 +922,12 @@ static JSValue defineProperties(JSGlobalObject* globalObject, JSObject* object, VM& vm = globalObject->vm(); auto scope = DECLARE_THROW_SCOPE(vm); +#if USE(BUN_JSC_ADDITIONS) + // entry.key() may be a fiber-word-tagged UniquedStringImpl*; DefaultRefDerefTraits::refIfNotNull would deref it. + Vector propertyNames; +#else Vector, 8> propertyNames; +#endif MarkedArgumentBuffer values; bool canUseFastPath = false; if (!hasIndexedProperties(properties->indexingType())) { @@ -1322,7 +1364,11 @@ static JSArray* getPropertyKeys(JSGlobalObject* globalObject, JSObject* object, ASSERT(!identifier.isPrivateName()); buffer[i].set(vm, owner, Symbol::create(vm, static_cast(*identifier.impl()))); } else +#if USE(BUN_JSC_ADDITIONS) + buffer[i].set(vm, owner, jsOwnedAtomBackedString(vm, identifier)); +#else buffer[i].set(vm, owner, jsOwnedString(vm, identifier.string())); +#endif } }; @@ -1359,7 +1405,11 @@ static JSArray* getPropertyKeys(JSGlobalObject* globalObject, JSObject* object, for (size_t i = 0; i < numProperties; i++) { const auto& identifier = properties[i]; ASSERT(!identifier.isSymbol()); +#if USE(BUN_JSC_ADDITIONS) + pushDirect(globalObject, keys, jsOwnedAtomBackedString(vm, identifier)); +#else pushDirect(globalObject, keys, jsOwnedString(vm, identifier.string())); +#endif RETURN_IF_EXCEPTION(scope, nullptr); } break; @@ -1383,7 +1433,11 @@ static JSArray* getPropertyKeys(JSGlobalObject* globalObject, JSObject* object, ASSERT(!identifier.isPrivateName()); pushDirect(globalObject, keys, Symbol::create(vm, static_cast(*identifier.impl()))); } else +#if USE(BUN_JSC_ADDITIONS) + pushDirect(globalObject, keys, jsOwnedAtomBackedString(vm, identifier)); +#else pushDirect(globalObject, keys, jsOwnedString(vm, identifier.string())); +#endif RETURN_IF_EXCEPTION(scope, nullptr); } break; diff --git a/Source/JavaScriptCore/runtime/ObjectPrototype.cpp b/Source/JavaScriptCore/runtime/ObjectPrototype.cpp index 9b1326dd5d514..9769e6e78d937 100644 --- a/Source/JavaScriptCore/runtime/ObjectPrototype.cpp +++ b/Source/JavaScriptCore/runtime/ObjectPrototype.cpp @@ -134,8 +134,19 @@ JSC_DEFINE_HOST_FUNCTION(objectProtoFuncHasOwnProperty, (JSGlobalObject* globalO Identifier propertyKey; SUPPRESS_UNCOUNTED_LOCAL UniquedStringImpl* uid = nullptr; if (subscript.isString()) { +#if USE(BUN_JSC_ADDITIONS) + if (uintptr_t fiber = asString(subscript)->tryGetCanonicalInlineFiberWord()) + uid = reinterpret_cast(fiber); + else { + propertyName = asString(subscript)->toAtomString(globalObject); + uid = propertyName.data; + if (uintptr_t fiber = Identifier::canonicalFiberWordFor(uid)) + uid = reinterpret_cast(fiber); + } +#else propertyName = asString(subscript)->toAtomString(globalObject); uid = propertyName.data; +#endif } else { propertyKey = subscript.toPropertyKey(globalObject); uid = propertyKey.impl(); diff --git a/Source/JavaScriptCore/runtime/OperationsInlines.h b/Source/JavaScriptCore/runtime/OperationsInlines.h index e310550b770c7..5596bf59e5371 100644 --- a/Source/JavaScriptCore/runtime/OperationsInlines.h +++ b/Source/JavaScriptCore/runtime/OperationsInlines.h @@ -158,6 +158,33 @@ ALWAYS_INLINE JSString* jsString(JSGlobalObject* globalObject, JSString* s1, JSS return nullptr; } +#if USE(BUN_JSC_ADDITIONS) + // Short concats: write both sides into a 16/24-byte inline cell instead of a + // 32-byte rope that would need resolving later. Only when both sides' + // characters are available without allocation (non-rope). + unsigned total = length1 + length2; + if (total <= JSString::maxBigInlineLength8 && !s1->isRope() && !s2->isRope()) { + auto view1 = s1->view(globalObject); + auto view2 = s2->view(globalObject); + if (view1->is8Bit() && view2->is8Bit()) { + Latin1Character buf[JSString::maxBigInlineLength8]; + memcpy(buf, view1->span8().data(), length1); + memcpy(buf + length1, view2->span8().data(), length2); + if (total <= JSString::maxInlineLength8) + return JSString::createInline8(vm, std::span { buf, total }); + return JSBigInlineString::create8(vm, std::span { buf, total }); + } + if (total <= JSString::maxBigInlineLength16) { + char16_t buf16[JSString::maxBigInlineLength16]; + view1->getCharacters(std::span { buf16, length1 }); + view2->getCharacters(std::span { buf16 + length1, length2 }); + if (total <= JSString::maxInlineLength16) + return JSString::createInline16(vm, std::span { buf16, total }); + return JSBigInlineString::create16(vm, std::span { buf16, total }); + } + } +#endif + return JSRopeString::create(vm, s1, s2); } @@ -184,6 +211,24 @@ ALWAYS_INLINE JSString* jsString(JSGlobalObject* globalObject, JSString* s1, JSS return nullptr; } +#if USE(BUN_JSC_ADDITIONS) + unsigned total = length1 + length2 + length3; + if (total <= JSString::maxBigInlineLength8 && !s1->isRope() && !s2->isRope() && !s3->isRope()) { + auto v1 = s1->view(globalObject); + auto v2 = s2->view(globalObject); + auto v3 = s3->view(globalObject); + if (v1->is8Bit() && v2->is8Bit() && v3->is8Bit()) { + Latin1Character buf[JSString::maxBigInlineLength8]; + memcpy(buf, v1->span8().data(), length1); + memcpy(buf + length1, v2->span8().data(), length2); + memcpy(buf + length1 + length2, v3->span8().data(), length3); + if (total <= JSString::maxInlineLength8) + return JSString::createInline8(vm, std::span { buf, total }); + return JSBigInlineString::create8(vm, std::span { buf, total }); + } + } +#endif + return JSRopeString::create(vm, s1, s2, s3); } diff --git a/Source/JavaScriptCore/runtime/PropertyName.h b/Source/JavaScriptCore/runtime/PropertyName.h index d56fbde7f3dc0..b8c4c528bec1a 100644 --- a/Source/JavaScriptCore/runtime/PropertyName.h +++ b/Source/JavaScriptCore/runtime/PropertyName.h @@ -31,6 +31,10 @@ #include "PrivateName.h" #include +#if USE(BUN_JSC_ADDITIONS) +#include "InlinePropertyKey.h" +#endif + WTF_ALLOW_UNSAFE_BUFFER_USAGE_BEGIN namespace JSC { @@ -72,12 +76,20 @@ class PropertyName { bool isSymbol() const { +#if USE(BUN_JSC_ADDITIONS) + return m_impl && uidIsSymbol(m_impl); +#else return m_impl && m_impl->isSymbol(); +#endif } bool isPrivateName() const { +#if USE(BUN_JSC_ADDITIONS) + return m_impl && !isInlinePropertyKey(m_impl) && m_impl->isSymbol() && static_cast(m_impl)->isPrivate(); +#else return isSymbol() && static_cast(m_impl)->isPrivate(); +#endif } UniquedStringImpl* uid() const @@ -87,15 +99,49 @@ class PropertyName { AtomStringImpl* publicName() const { +#if USE(BUN_JSC_ADDITIONS) + if (!m_impl) + return nullptr; + if (isInlinePropertyKey(m_impl)) [[unlikely]] { + // Callers (Bun bindings, Lookup.cpp, JSCustom*Function) assume + // "not symbol ⇒ non-null". Materialize the atom; the leaked ref + // just pins one interned 2..5-char string — pre-fiber behaviour. + uintptr_t w = reinterpret_cast(m_impl); + unsigned len = inlinePropertyKeyLength(w); + const uint8_t* bytes = reinterpret_cast(&w); + RefPtr atom = inlinePropertyKeyIs8Bit(w) + ? AtomStringImpl::add(std::span { bytes + 1, len }) + : AtomStringImpl::add(std::span { reinterpret_cast(bytes + 2), len }); + return atom.leakRef(); + } + return m_impl->isSymbol() ? nullptr : static_cast(m_impl); +#else return (!m_impl || m_impl->isSymbol()) ? nullptr : static_cast(m_impl); +#endif } void dump(PrintStream& out) const { +#if USE(BUN_JSC_ADDITIONS) + if (m_impl) { + if (isInlinePropertyKey(m_impl)) { + uintptr_t word = reinterpret_cast(m_impl); + unsigned len = inlinePropertyKeyLength(word); + const uint8_t* bytes = reinterpret_cast(&word); + if (inlinePropertyKeyIs8Bit(word)) + out.print(StringView(std::span { bytes + 1, len })); + else + out.print(StringView(std::span { reinterpret_cast(bytes + 2), len })); + } else + out.print(m_impl); + } else + out.print(""); +#else if (m_impl) out.print(m_impl); else out.print(""); +#endif } private: @@ -115,17 +161,56 @@ inline bool operator==(PropertyName a, PropertyName b) inline bool operator==(PropertyName a, const char* b) { +#if USE(BUN_JSC_ADDITIONS) + auto* uid = a.uid(); + if (isInlinePropertyKey(uid)) { + if (!b) + return false; + uintptr_t w = reinterpret_cast(uid); + unsigned len = inlinePropertyKeyLength(w); + const uint8_t* bytes = reinterpret_cast(&w); + if (inlinePropertyKeyIs8Bit(w)) { + const Latin1Character* chars = bytes + 1; + for (unsigned i = 0; i < len; ++i) { + if (!b[i] || static_cast(b[i]) != chars[i]) + return false; + } + return !b[len]; + } + const char16_t* chars = reinterpret_cast(bytes + 2); + for (unsigned i = 0; i < len; ++i) { + if (!b[i] || static_cast(static_cast(b[i])) != chars[i]) + return false; + } + return !b[len]; + } + return equal(uid, b); +#else return equal(a.uid(), b); +#endif } ALWAYS_INLINE std::optional parseIndex(PropertyName propertyName) { +#if USE(BUN_JSC_ADDITIONS) + auto uid = propertyName.uid(); + if (!uid || uidIsSymbol(uid)) + return std::nullopt; + if (isInlinePropertyKey(uid)) { + uintptr_t w = reinterpret_cast(uid); + if (inlinePropertyKeyIs8Bit(w)) + return parseIndex(inlinePropertyKeySpan8(w)); + return std::nullopt; + } + return parseIndex(*uid); +#else auto uid = propertyName.uid(); if (!uid) return std::nullopt; if (uid->isSymbol()) return std::nullopt; return parseIndex(*uid); +#endif } template @@ -157,6 +242,39 @@ ALWAYS_INLINE bool isCanonicalNumericIndexString(UniquedStringImpl* propertyName { if (!propertyName) return false; +#if USE(BUN_JSC_ADDITIONS) + if (isInlinePropertyKey(propertyName)) { + uintptr_t w = reinterpret_cast(propertyName); + unsigned len = inlinePropertyKeyLength(w); + if (!len) + return false; + const uint8_t* bytes = reinterpret_cast(&w); + if (inlinePropertyKeyIs8Bit(w)) { + std::span chars { bytes + 1, len }; + auto fastResult = fastIsCanonicalNumericIndexString(chars); + if (fastResult) + return *fastResult; + double index = jsToNumber(StringView(chars)); + NumberToStringBuffer buffer; + auto out = WTF::numberToStringAndSize(index, buffer); + return WTF::equal(chars, byteCast(out)); + } + std::span chars { reinterpret_cast(bytes + 2), len }; + auto fastResult = fastIsCanonicalNumericIndexString(chars); + if (fastResult) + return *fastResult; + double index = jsToNumber(StringView(chars)); + NumberToStringBuffer buffer; + auto out = byteCast(WTF::numberToStringAndSize(index, buffer)); + if (chars.size() != out.size()) + return false; + for (size_t i = 0; i < out.size(); ++i) { + if (chars[i] != static_cast(out[i])) + return false; + } + return true; + } +#endif if (propertyName->isSymbol()) return false; if (!propertyName->length()) diff --git a/Source/JavaScriptCore/runtime/PropertyNameArray.h b/Source/JavaScriptCore/runtime/PropertyNameArray.h index 5315af4e6e1e3..725c16d0efcde 100644 --- a/Source/JavaScriptCore/runtime/PropertyNameArray.h +++ b/Source/JavaScriptCore/runtime/PropertyNameArray.h @@ -138,6 +138,10 @@ ALWAYS_INLINE void PropertyNameArrayBuilder::add(UniquedStringImpl* identifier) ALWAYS_INLINE bool PropertyNameArrayBuilder::isUidMatchedToTypeMode(UniquedStringImpl* identifier) { +#if USE(BUN_JSC_ADDITIONS) + if (isInlinePropertyKey(identifier)) + return includeStringProperties(); +#endif if (identifier->isSymbol()) { if (!includeSymbolProperties()) return false; diff --git a/Source/JavaScriptCore/runtime/PropertyTable.cpp b/Source/JavaScriptCore/runtime/PropertyTable.cpp index e059be337f4f1..59176b5ee20cb 100644 --- a/Source/JavaScriptCore/runtime/PropertyTable.cpp +++ b/Source/JavaScriptCore/runtime/PropertyTable.cpp @@ -85,7 +85,11 @@ PropertyTable::PropertyTable(VM& vm, const PropertyTable& other) memcpy(std::bit_cast(m_indexVector & indexVectorMask), std::bit_cast(other.m_indexVector & indexVectorMask), dataSize(isCompact())); forEachProperty([&](auto& entry) { +#if USE(BUN_JSC_ADDITIONS) + uidRef(entry.key()); +#else entry.key()->ref(); +#endif return IterationStatus::Continue; }); @@ -114,7 +118,11 @@ PropertyTable::PropertyTable(VM& vm, unsigned initialCapacity, const PropertyTab other.forEachProperty([&](auto& entry) { ASSERT(canInsert(entry)); reinsert(vector, table, entry); +#if USE(BUN_JSC_ADDITIONS) + uidRef(entry.key()); +#else entry.key()->ref(); +#endif return IterationStatus::Continue; }); }); @@ -150,7 +158,11 @@ void PropertyTable::destroy(JSCell* cell) PropertyTable::~PropertyTable() { forEachProperty([&](auto& entry) { +#if USE(BUN_JSC_ADDITIONS) + uidDeref(entry.key()); +#else entry.key()->deref(); +#endif return IterationStatus::Continue; }); destroyIndexVector(m_indexVector); diff --git a/Source/JavaScriptCore/runtime/PropertyTable.h b/Source/JavaScriptCore/runtime/PropertyTable.h index fbbe734b1f83c..d91694f5b6fd9 100644 --- a/Source/JavaScriptCore/runtime/PropertyTable.h +++ b/Source/JavaScriptCore/runtime/PropertyTable.h @@ -30,6 +30,9 @@ #include #include +#if USE(BUN_JSC_ADDITIONS) +#include "InlinePropertyKey.h" +#endif #define DUMP_PROPERTYMAP_STATS 0 #define DUMP_PROPERTYMAP_COLLISIONS 0 @@ -303,6 +306,9 @@ PropertyTable::FindResult PropertyTable::findImpl(const Index* indexVector, cons unsigned indexMask = m_indexMask; unsigned probeCount = 0; unsigned index = hash & indexMask; +#if USE(BUN_JSC_ADDITIONS) && ASSERT_ENABLED + bool keyIsFiber = isInlinePropertyKey(key); +#endif #if DUMP_PROPERTYMAP_STATS ++propertyTableStats->numFinds; @@ -317,6 +323,22 @@ PropertyTable::FindResult PropertyTable::findImpl(const Index* indexVector, cons ASSERT(!m_deletedOffsets || !m_deletedOffsets->contains(entry.offset())); return FindResult { entryIndex, index, entry.offset(), entry.attributes() }; } +#if USE(BUN_JSC_ADDITIONS) && ASSERT_ENABLED + // D.4 single-rep coherence guarantees one uid per key content (fiber XOR atom). + // A cross-rep content match means a producer bypassed canonicalFiberWordFor — + // assert in debug so it gets fixed; release builds pay nothing for this check. + else { + UniquedStringImpl* entryKey = entry.key(); + bool entryIsFiber = isInlinePropertyKey(entryKey); + if (keyIsFiber != entryIsFiber) [[unlikely]] { + const UniquedStringImpl* implSide = keyIsFiber ? entryKey : key; + uintptr_t fiberWord = inlinePropertyKeyWord(keyIsFiber ? key : entryKey); + if (implSide && !implSide->isSymbol() && inlinePropertyKeyIs8Bit(fiberWord) + && WTF::equal(implSide, inlinePropertyKeySpan8(fiberWord))) + ASSERT_NOT_REACHED_WITH_MESSAGE("dual-rep key reached PropertyTable"); + } + } +#endif #if DUMP_PROPERTYMAP_STATS ++propertyTableStats->numCollisions; @@ -335,7 +357,11 @@ PropertyTable::FindResult PropertyTable::findImpl(const Index* indexVector, cons inline PropertyTable::FindResult PropertyTable::find(const KeyType& key) { ASSERT(key); +#if USE(BUN_JSC_ADDITIONS) + ASSERT(isInlinePropertyKey(key) || key->isAtom() || key->isSymbol()); +#else ASSERT(key->isAtom() || key->isSymbol()); +#endif return withIndexVector([&](auto* vector) { return findImpl(vector, tableFromIndexVector(vector), key); }); @@ -344,7 +370,11 @@ inline PropertyTable::FindResult PropertyTable::find(const KeyType& key) inline std::tuple PropertyTable::get(const KeyType& key) { ASSERT(key); +#if USE(BUN_JSC_ADDITIONS) + ASSERT(isInlinePropertyKey(key) || key->isAtom() || key->isSymbol()); +#else ASSERT(key->isAtom() || key->isSymbol()); +#endif ASSERT(key != PROPERTY_MAP_DELETED_ENTRY_KEY); if (!m_keyCount) @@ -372,7 +402,11 @@ ALWAYS_INLINE std::tuple PropertyTable::addAfter #endif // Ref the key +#if USE(BUN_JSC_ADDITIONS) + uidRef(entry.key()); +#else entry.key()->ref(); +#endif // ensure capacity is available. if (!canInsert(entry)) { @@ -408,7 +442,11 @@ inline void PropertyTable::remove(VM& vm, KeyType key, unsigned entryIndex, unsi vector[index] = deletedEntryIndex(); tableFromIndexVector(vector)[entryIndex - 1].setKey(PROPERTY_MAP_DELETED_ENTRY_KEY); }); +#if USE(BUN_JSC_ADDITIONS) + uidDeref(key); +#else key->deref(); +#endif ASSERT(m_keyCount >= 1); --m_keyCount; diff --git a/Source/JavaScriptCore/runtime/RegExpConstructor.cpp b/Source/JavaScriptCore/runtime/RegExpConstructor.cpp index 830890b0d5708..8223cb1e3350b 100644 --- a/Source/JavaScriptCore/runtime/RegExpConstructor.cpp +++ b/Source/JavaScriptCore/runtime/RegExpConstructor.cpp @@ -187,7 +187,13 @@ JSC_DEFINE_CUSTOM_GETTER(regExpConstructorDollar, (JSGlobalObject* globalObject, auto scope = DECLARE_THROW_SCOPE(vm); if (JSValue::decode(thisValue) != globalObject->regExpConstructor()) return throwVMTypeError(globalObject, scope, "RegExp.$N getters require RegExp constructor as |this|"_s); +#if USE(BUN_JSC_ADDITIONS) + // "$1".."$9" are 2-char fiber-word keys; second char is at payload byte 2. + auto* uid = propertyName.uid(); + unsigned N = (isInlinePropertyKey(uid) ? inlinePropertyKeySpan8(reinterpret_cast(uid))[1] : uid->at(1)) - '0'; +#else unsigned N = propertyName.uid()->at(1) - '0'; +#endif ASSERT(N >= 1 && N <= 9); RELEASE_AND_RETURN(scope, JSValue::encode(globalObject->regExpGlobalData().getBackref(globalObject, N))); } diff --git a/Source/JavaScriptCore/runtime/RegExpPrototype.cpp b/Source/JavaScriptCore/runtime/RegExpPrototype.cpp index 28a905f9fdd6b..dfc1d14c4ea48 100644 --- a/Source/JavaScriptCore/runtime/RegExpPrototype.cpp +++ b/Source/JavaScriptCore/runtime/RegExpPrototype.cpp @@ -165,7 +165,11 @@ JSC_DEFINE_HOST_FUNCTION(regExpProtoFuncTest, (JSGlobalObject* globalObject, Cal auto* regExp = dynamicDowncast(thisValue); if (!regExp) [[unlikely]] return throwVMTypeError(globalObject, scope, "Builtin RegExp exec can only be called on a RegExp object"_s); +#if USE(BUN_JSC_ADDITIONS) + auto strValue = str->view(globalObject); +#else auto strValue = str->value(globalObject); +#endif RETURN_IF_EXCEPTION(scope, { }); if (!strValue->isNull() && regExp->getLastIndex().isNumber()) [[likely]] RELEASE_AND_RETURN(scope, JSValue::encode(jsBoolean(regExp->test(globalObject, str)))); @@ -209,6 +213,19 @@ static JSValue regExpMatchSlow(JSGlobalObject* globalObject, JSObject* thisObjec // 4. Let flags be ? ToString(? Get(regexp, "flags")). JSValue flagsValue = thisObject->get(globalObject, vm.propertyNames->flags); RETURN_IF_EXCEPTION(scope, { }); +#if USE(BUN_JSC_ADDITIONS) + auto* flagsJSString = flagsValue.toString(globalObject); + RETURN_IF_EXCEPTION(scope, { }); + auto flags = flagsJSString->view(globalObject); + RETURN_IF_EXCEPTION(scope, { }); + + // 5. If flags does not contain "g", return ? RegExpExec(regexp, string). + if (!flags->contains('g')) + RELEASE_AND_RETURN(scope, regExpExec(globalObject, thisObject, string)); + + // 6. If flags contains "u" or flags contains "v", let fullUnicode be true; else let fullUnicode be false. + bool fullUnicode = flags->contains('u') || flags->contains('v'); +#else String flags = flagsValue.toWTFString(globalObject); RETURN_IF_EXCEPTION(scope, { }); @@ -218,6 +235,7 @@ static JSValue regExpMatchSlow(JSGlobalObject* globalObject, JSObject* thisObjec // 6. If flags contains "u" or flags contains "v", let fullUnicode be true; else let fullUnicode be false. bool fullUnicode = flags.contains('u') || flags.contains('v'); +#endif // 7. Perform ? Set(regexp, "lastIndex", +0𝔽, true). PutPropertySlot lastIndexSlot(thisObject, true); @@ -335,8 +353,21 @@ JSC_DEFINE_HOST_FUNCTION(regExpProtoFuncCompile, (JSGlobalObject* globalObject, String pattern = arg0.isUndefined() ? emptyString() : arg0.toWTFString(globalObject); RETURN_IF_EXCEPTION(scope, encodedJSValue()); +#if USE(BUN_JSC_ADDITIONS) + std::optional> flags; + if (arg1.isUndefined()) + flags = OptionSet { }; + else { + auto* arg1String = arg1.toString(globalObject); + RETURN_IF_EXCEPTION(scope, encodedJSValue()); + auto arg1View = arg1String->view(globalObject); + RETURN_IF_EXCEPTION(scope, encodedJSValue()); + flags = Yarr::parseFlags(StringView(arg1View)); + } +#else auto flags = arg1.isUndefined() ? std::make_optional(OptionSet { }) : Yarr::parseFlags(arg1.toWTFString(globalObject)); RETURN_IF_EXCEPTION(scope, encodedJSValue()); +#endif if (!flags) return throwVMError(globalObject, scope, createSyntaxError(globalObject, "Invalid flags supplied to RegExp constructor."_s)); @@ -419,6 +450,21 @@ JSC_DEFINE_HOST_FUNCTION(regExpProtoFuncToString, (JSGlobalObject* globalObject, JSValue sourceValue = thisObject->get(globalObject, vm.propertyNames->source); RETURN_IF_EXCEPTION(scope, { }); +#if USE(BUN_JSC_ADDITIONS) + auto* sourceJSString = sourceValue.toString(globalObject); + RETURN_IF_EXCEPTION(scope, { }); + auto source = sourceJSString->view(globalObject); + RETURN_IF_EXCEPTION(scope, { }); + + JSValue flagsValue = thisObject->get(globalObject, vm.propertyNames->flags); + RETURN_IF_EXCEPTION(scope, { }); + auto* flagsJSString = flagsValue.toString(globalObject); + RETURN_IF_EXCEPTION(scope, { }); + auto flags = flagsJSString->view(globalObject); + RETURN_IF_EXCEPTION(scope, { }); + + RELEASE_AND_RETURN(scope, JSValue::encode(jsMakeNontrivialString(globalObject, '/', StringView(source), '/', StringView(flags)))); +#else String source = sourceValue.toWTFString(globalObject); RETURN_IF_EXCEPTION(scope, { }); @@ -428,6 +474,7 @@ JSC_DEFINE_HOST_FUNCTION(regExpProtoFuncToString, (JSGlobalObject* globalObject, RETURN_IF_EXCEPTION(scope, { }); RELEASE_AND_RETURN(scope, JSValue::encode(jsMakeNontrivialString(globalObject, '/', source, '/', flags))); +#endif } JSC_DEFINE_HOST_FUNCTION(regExpProtoGetterGlobal, (JSGlobalObject* globalObject, CallFrame* callFrame)) @@ -1030,6 +1077,20 @@ JSValue regExpSplitSlow(JSGlobalObject* globalObject, JSObject* thisObject, JSSt // 5. Let flags be ? ToString(? Get(regexp, "flags")). JSValue flagsValue = thisObject->get(globalObject, vm.propertyNames->flags); RETURN_IF_EXCEPTION(scope, { }); +#if USE(BUN_JSC_ADDITIONS) + auto* flagsJSString = flagsValue.toString(globalObject); + RETURN_IF_EXCEPTION(scope, { }); + auto flags = flagsJSString->view(globalObject); + RETURN_IF_EXCEPTION(scope, { }); + + // 6. If flags contains "u" or flags contains "v", let unicodeMatching be true. + // 7. Else, let unicodeMatching be false. + bool unicodeMatching = flags->contains('u') || flags->contains('v'); + + // 8. If flags contains "y", let newFlags be flags. + // 9. Else, let newFlags be the string-concatenation of flags and "y". + String newFlags = flags->contains('y') ? flags->toString() : tryMakeString(StringView(flags), 'y'); +#else String flags = flagsValue.toWTFString(globalObject); RETURN_IF_EXCEPTION(scope, { }); @@ -1040,6 +1101,7 @@ JSValue regExpSplitSlow(JSGlobalObject* globalObject, JSObject* thisObject, JSSt // 8. If flags contains "y", let newFlags be flags. // 9. Else, let newFlags be the string-concatenation of flags and "y". String newFlags = flags.contains('y') ? flags : tryMakeString(flags, 'y'); +#endif if (newFlags.isNull()) [[unlikely]] { throwOutOfMemoryError(globalObject, scope); return { }; @@ -1264,9 +1326,17 @@ static inline String getSubstitution(JSGlobalObject* globalObject, const String& JSValue capture = namedCaptures->get(globalObject, Identifier::fromString(vm, groupName)); RETURN_IF_EXCEPTION(scope, String()); if (!capture.isUndefined()) { +#if USE(BUN_JSC_ADDITIONS) + auto* captureJSString = capture.toString(globalObject); + RETURN_IF_EXCEPTION(scope, String()); + auto captureString = captureJSString->view(globalObject); + RETURN_IF_EXCEPTION(scope, String()); + result.append(StringView(captureString)); +#else String captureString = capture.toWTFString(globalObject); RETURN_IF_EXCEPTION(scope, String()); result.append(captureString); +#endif } start = groupNameEndIndex + 1; break; @@ -1349,6 +1419,22 @@ JSValue regExpReplaceGeneric(JSGlobalObject* globalObject, JSObject* thisObject, // 7. Let flags be ? ToString(? Get(rx, "flags")). JSValue flagsValue = thisObject->get(globalObject, vm.propertyNames->flags); RETURN_IF_EXCEPTION(scope, { }); +#if USE(BUN_JSC_ADDITIONS) + auto* flagsJSString = flagsValue.toString(globalObject); + RETURN_IF_EXCEPTION(scope, { }); + auto flags = flagsJSString->view(globalObject); + RETURN_IF_EXCEPTION(scope, { }); + + // 8. If flags contains "g", let global be true. Else, let global be false. + bool global = flags->contains('g'); + + // 9. If global is true, then + // a. If flags contains "u" or "v", let fullUnicode be true. Else, let fullUnicode be false. + // b. Perform ? Set(rx, "lastIndex", +0F, true). + bool fullUnicode = false; + if (global) { + fullUnicode = flags->contains('u') || flags->contains('v'); +#else String flags = flagsValue.toWTFString(globalObject); RETURN_IF_EXCEPTION(scope, { }); @@ -1361,6 +1447,7 @@ JSValue regExpReplaceGeneric(JSGlobalObject* globalObject, JSObject* thisObject, bool fullUnicode = false; if (global) { fullUnicode = flags.contains('u') || flags.contains('v'); +#endif PutPropertySlot slot(thisObject, true); thisObject->methodTable()->put(thisObject, globalObject, vm.propertyNames->lastIndex, jsNumber(0), slot); RETURN_IF_EXCEPTION(scope, { }); @@ -1399,6 +1486,22 @@ JSValue regExpReplaceGeneric(JSGlobalObject* globalObject, JSObject* thisObject, JSObject* resultObject = asObject(result); JSValue matchValue = resultObject->get(globalObject, static_cast(0)); RETURN_IF_EXCEPTION(scope, { }); +#if USE(BUN_JSC_ADDITIONS) + // phase-3 audit: all 9 view() swaps of 96decd2dc1 here are KEEP — none sit + // in JetStream's fast path (stringProtoFuncReplace → replaceUsingRegExpSearch, + // not this slow generic). −13% regexp regression lives in StringPrototype.cpp + // replaceUsingRegExpSearch / ed331542f6 jsSubstringOfResolved, not this file. + // length() below is a harmless local micro-opt, not the regression fix. + auto* matchJSString = matchValue.toString(globalObject); + RETURN_IF_EXCEPTION(scope, { }); + + // 2. If matchStr is the empty String, then + // a. Let thisIndex be R(? ToLength(? Get(rx, "lastIndex"))). + // b. If flags contains "u" or flags contains "v", let fullUnicode be true; otherwise let fullUnicode be false. + // c. Let nextIndex be AdvanceStringIndex(S, thisIndex, fullUnicode). + // d. Perform ? Set(rx, "lastIndex", F(nextIndex), true). + if (!matchJSString->length()) { +#else String matchStr = matchValue.toWTFString(globalObject); RETURN_IF_EXCEPTION(scope, { }); @@ -1408,6 +1511,7 @@ JSValue regExpReplaceGeneric(JSGlobalObject* globalObject, JSObject* thisObject, // c. Let nextIndex be AdvanceStringIndex(S, thisIndex, fullUnicode). // d. Perform ? Set(rx, "lastIndex", F(nextIndex), true). if (matchStr.isEmpty()) { +#endif JSValue lastIndexValue = thisObject->get(globalObject, vm.propertyNames->lastIndex); RETURN_IF_EXCEPTION(scope, { }); uint64_t thisIndex = lastIndexValue.toLength(globalObject); diff --git a/Source/JavaScriptCore/runtime/StackFrame.cpp b/Source/JavaScriptCore/runtime/StackFrame.cpp index c066acca95a27..d1990db85b035 100644 --- a/Source/JavaScriptCore/runtime/StackFrame.cpp +++ b/Source/JavaScriptCore/runtime/StackFrame.cpp @@ -222,8 +222,25 @@ String StackFrame::functionName(VM& vm) const if (jsFrame.callee && jsFrame.callee->isObject()) name = getCalculatedDisplayName(vm, uncheckedDowncast(jsFrame.callee.get())).impl(); else if (jsFrame.codeBlock) { - if (auto* executable = dynamicDowncast(jsFrame.codeBlock->ownerExecutable())) + if (auto* executable = dynamicDowncast(jsFrame.codeBlock->ownerExecutable())) { +#if USE(BUN_JSC_ADDITIONS) + // ecmaName().impl() may be a fiber-word-tagged pointer; decode + // instead of letting String(StringImpl*) ref() a non-pointer. + UniquedStringImpl* uid = executable->ecmaName().impl(); + if (isInlinePropertyKey(uid)) { + uintptr_t word = inlinePropertyKeyWord(uid); + unsigned len = inlinePropertyKeyLength(word); + const uint8_t* bytes = reinterpret_cast(&word); + if (inlinePropertyKeyIs8Bit(word)) + name = String(std::span { bytes + 1, len }); + else + name = String(std::span { reinterpret_cast(bytes + 2), len }); + } else + name = uid; +#else name = executable->ecmaName().impl(); +#endif + } } if (name.isNull()) diff --git a/Source/JavaScriptCore/runtime/StringConstructor.cpp b/Source/JavaScriptCore/runtime/StringConstructor.cpp index ebfe1a10c92cb..e50510142b682 100644 --- a/Source/JavaScriptCore/runtime/StringConstructor.cpp +++ b/Source/JavaScriptCore/runtime/StringConstructor.cpp @@ -24,6 +24,9 @@ #include "JSCInlines.h" #include "StringPrototype.h" #include +#if USE(BUN_JSC_ADDITIONS) +#include "JSStringBuilder.h" +#endif namespace JSC { @@ -120,7 +123,11 @@ JSC_DEFINE_HOST_FUNCTION(stringFromCodePoint, (JSGlobalObject* globalObject, Cal auto scope = DECLARE_THROW_SCOPE(vm); unsigned length = callFrame->argumentCount(); +#if USE(BUN_JSC_ADDITIONS) + JSStringBuilder builder; +#else StringBuilder builder; +#endif builder.reserveCapacity(length); for (unsigned i = 0; i < length; ++i) { @@ -140,7 +147,11 @@ JSC_DEFINE_HOST_FUNCTION(stringFromCodePoint, (JSGlobalObject* globalObject, Cal } } +#if USE(BUN_JSC_ADDITIONS) + RELEASE_AND_RETURN(scope, JSValue::encode(builder.toJS(vm))); +#else RELEASE_AND_RETURN(scope, JSValue::encode(jsString(vm, builder.toString()))); +#endif } JSString* stringFromCodePoint(JSGlobalObject* globalObject, int32_t arg) diff --git a/Source/JavaScriptCore/runtime/StringPrototype.cpp b/Source/JavaScriptCore/runtime/StringPrototype.cpp index ec737caf620eb..34994a4fe9db8 100644 --- a/Source/JavaScriptCore/runtime/StringPrototype.cpp +++ b/Source/JavaScriptCore/runtime/StringPrototype.cpp @@ -706,10 +706,19 @@ JSC_DEFINE_HOST_FUNCTION(stringProtoFuncReplaceAll, (JSGlobalObject* globalObjec JSValue flagsValue = asObject(searchValue)->get(globalObject, vm.propertyNames->flags); RETURN_IF_EXCEPTION(scope, { }); +#if USE(BUN_JSC_ADDITIONS) + auto* flagsJSString = flagsValue.toString(globalObject); + RETURN_IF_EXCEPTION(scope, { }); + auto flags = flagsJSString->view(globalObject); + RETURN_IF_EXCEPTION(scope, { }); + if (!flags->contains('g')) [[unlikely]] + return throwVMTypeError(globalObject, scope, "String.prototype.replaceAll argument must not be a non-global regular expression"_s); +#else String flags = flagsValue.toWTFString(globalObject); RETURN_IF_EXCEPTION(scope, { }); if (!flags.contains('g')) [[unlikely]] return throwVMTypeError(globalObject, scope, "String.prototype.replaceAll argument must not be a non-global regular expression"_s); +#endif } JSObject* searchObject = asObject(searchValue); @@ -822,6 +831,36 @@ JSC_DEFINE_HOST_FUNCTION(stringProtoFuncCodePointAt, (JSGlobalObject* globalObje if (!checkObjectCoercible(thisValue)) [[unlikely]] return throwVMTypeError(globalObject, scope); +#if USE(BUN_JSC_ADDITIONS) + // Fast path for inline small strings: read the code unit from the cell + // bytes without materializing a StringImpl. + if (thisValue.isString() && asString(thisValue)->isInline()) { + auto* jsString = asString(thisValue); + auto view = jsString->view(globalObject); + RETURN_IF_EXCEPTION(scope, encodedJSValue()); + unsigned length = view->length(); + JSValue argument0 = callFrame->argument(0); + unsigned position; + if (argument0.isUInt32()) + position = argument0.asUInt32(); + else { + double d = argument0.toIntegerOrInfinity(globalObject); + RETURN_IF_EXCEPTION(scope, encodedJSValue()); + if (!(d >= 0 && d < length)) + return JSValue::encode(jsUndefined()); + position = static_cast(d); + } + if (position >= length) + return JSValue::encode(jsUndefined()); + if (view->is8Bit()) + return JSValue::encode(jsNumber(static_cast(view->span8()[position]))); + char32_t c; + auto chars = view->span16(); + U16_NEXT(chars, position, length, c); + return JSValue::encode(jsNumber(c)); + } +#endif + String string = thisValue.toWTFString(globalObject); // Intentionally resolving as codePointAt requires resolved strings in the higher tiers. RETURN_IF_EXCEPTION(scope, encodedJSValue()); unsigned length = string.length(); @@ -1115,9 +1154,22 @@ JSCell* stringSplitFast(JSGlobalObject* globalObject, JSString* thisString, JSSt auto identifier = subView.is8Bit() ? Identifier::fromString(vm, subView.span8()) : Identifier::fromString(vm, subView.span16()); DeferGC defer(vm); +#if USE(BUN_JSC_ADDITIONS) + // atomStringToJSStringMap's contract is AtomStringImpl*-identity (its + // only reader, arrayProtoFuncIndexOf, compares getValueImpl() against a + // toAtomString() result). Identifier::fromString fiber-words 2..5-char + // names, so impl() is a tagged word there; this is a site that needs + // the AtomStringImpl* specifically, so route through string() — which + // the lambda already does for the JSString payload. + AtomString atom = identifier.string(); + string = vm.atomStringToJSStringMap.ensureValue(atom.impl(), [&] { + return jsString(vm, atom); + }); +#else string = vm.atomStringToJSStringMap.ensureValue(identifier.impl(), [&] { return jsString(vm, identifier.string()); }); +#endif } else { string = jsSubstring(globalObject, thisString, start, end - start); RETURN_IF_EXCEPTION(scope, { }); @@ -1575,10 +1627,19 @@ JSC_DEFINE_HOST_FUNCTION(stringProtoFuncMatchAll, (JSGlobalObject* globalObject, if (isArgRegExp) { JSValue flagsValue = asObject(regexpValue)->get(globalObject, vm.propertyNames->flags); RETURN_IF_EXCEPTION(scope, { }); +#if USE(BUN_JSC_ADDITIONS) + auto* flagsJSString = flagsValue.toString(globalObject); + RETURN_IF_EXCEPTION(scope, { }); + auto flags = flagsJSString->view(globalObject); + RETURN_IF_EXCEPTION(scope, { }); + if (!flags->contains('g')) [[unlikely]] + return throwVMTypeError(globalObject, scope, "String.prototype.matchAll argument must not be a non-global regular expression"_s); +#else String flags = flagsValue.toWTFString(globalObject); RETURN_IF_EXCEPTION(scope, { }); if (!flags.contains('g')) [[unlikely]] return throwVMTypeError(globalObject, scope, "String.prototype.matchAll argument must not be a non-global regular expression"_s); +#endif } JSValue matcher = asObject(regexpValue)->get(globalObject, vm.propertyNames->matchAllSymbol); @@ -1923,6 +1984,50 @@ static inline JSValue trimString(JSGlobalObject* globalObject, JSValue thisValue if (!checkObjectCoercible(thisValue)) [[unlikely]] return throwTypeError(globalObject, scope); +#if USE(BUN_JSC_ADDITIONS) + auto* jsStr = thisValue.toString(globalObject); + RETURN_IF_EXCEPTION(scope, { }); + auto str = jsStr->view(globalObject); + RETURN_IF_EXCEPTION(scope, { }); + + unsigned length = str->length(); + if (!length) [[unlikely]] { + if (thisValue.isString()) + return thisValue; + RELEASE_AND_RETURN(scope, jsEmptyString(vm)); + } + + unsigned left = 0; + unsigned right = length; + + if (str->is8Bit()) { + auto characters = str->span8(); + if constexpr (static_cast(trimKind) & static_cast(TrimKind::TrimStart)) { + while (left < length && isStrWhiteSpace(characters[left])) + left++; + } + if constexpr (static_cast(trimKind) & static_cast(TrimKind::TrimEnd)) { + while (right > left && isStrWhiteSpace(characters[right - 1])) + right--; + } + } else { + auto characters = str->span16(); + if constexpr (static_cast(trimKind) & static_cast(TrimKind::TrimStart)) { + while (left < length && isStrWhiteSpace(characters[left])) + left++; + } + if constexpr (static_cast(trimKind) & static_cast(TrimKind::TrimEnd)) { + while (right > left && isStrWhiteSpace(characters[right - 1])) + right--; + } + } + + // Don't gc allocate a new string if we don't have to. + if (!left && right == length && thisValue.isString()) + return thisValue; + + RELEASE_AND_RETURN(scope, jsSubstring(globalObject, vm, jsStr, left, right - left)); +#else String str = thisValue.toWTFString(globalObject); RETURN_IF_EXCEPTION(scope, { }); @@ -1963,6 +2068,7 @@ static inline JSValue trimString(JSGlobalObject* globalObject, JSValue thisValue return thisValue; RELEASE_AND_RETURN(scope, jsString(vm, str.substringSharingImpl(left, right - left))); +#endif } JSC_DEFINE_HOST_FUNCTION(stringProtoFuncTrim, (JSGlobalObject* globalObject, CallFrame* callFrame)) @@ -2332,7 +2438,11 @@ JSC_DEFINE_HOST_FUNCTION(stringProtoFuncToWellFormed, (JSGlobalObject* globalObj if (stringValue->is8Bit()) return JSValue::encode(stringValue); +#if USE(BUN_JSC_ADDITIONS) + auto string = stringValue->view(globalObject); +#else auto string = stringValue->value(globalObject); +#endif RETURN_IF_EXCEPTION(scope, { }); if (string->is8Bit()) diff --git a/Source/JavaScriptCore/runtime/StringPrototypeInlines.h b/Source/JavaScriptCore/runtime/StringPrototypeInlines.h index 80090d1170020..3471ce7fc3bd5 100644 --- a/Source/JavaScriptCore/runtime/StringPrototypeInlines.h +++ b/Source/JavaScriptCore/runtime/StringPrototypeInlines.h @@ -537,11 +537,19 @@ inline JSString* tryReplaceOneCharUsingString(JSGlobalObject* globalObject, JSSt return nullptr; RETURN_IF_EXCEPTION(scope, nullptr); +#if USE(BUN_JSC_ADDITIONS) + auto searchString = search->view(globalObject); +#else auto searchString = search->value(globalObject); +#endif if (searchString->length() != 1) RELEASE_AND_RETURN(scope, nullptr); +#if USE(BUN_JSC_ADDITIONS) + auto replaceString = replacement->view(globalObject); +#else auto replaceString = replacement->value(globalObject); +#endif RETURN_IF_EXCEPTION(scope, nullptr); if constexpr (check == DollarCheck::Yes) { if (replaceString->find('$') != notFound) @@ -854,7 +862,13 @@ static ALWAYS_INLINE JSString* replaceAllWithCacheUsingRegExpSearchThreeArgument auto jsString = jsResult.toString(globalObject); RETURN_IF_EXCEPTION_WITH_TRAPS_DEFERRED(scope, nullptr); +#if USE(BUN_JSC_ADDITIONS) + // Only is8Bit()+length() are read; use view() so inline results + // are not materialized to a StringImpl here. + auto string = jsString->view(globalObject); +#else auto string = jsString->value(globalObject); +#endif RETURN_IF_EXCEPTION_WITH_TRAPS_DEFERRED(scope, nullptr); replacementsAre8Bit &= string->is8Bit(); @@ -890,8 +904,13 @@ static ALWAYS_INLINE JSString* replaceAllWithCacheUsingRegExpSearchThreeArgument substring.getCharacters8(buffer.subspan(bufferPos)); bufferPos += substring.length(); +#if USE(BUN_JSC_ADDITIONS) + auto replacement = asString(slot)->view(globalObject); + StringView { replacement }.getCharacters8(buffer.subspan(bufferPos)); +#else auto replacement = asString(slot)->value(globalObject); StringView { replacement }.getCharacters8(buffer.subspan(bufferPos)); +#endif bufferPos += replacement->length(); ++index; @@ -925,8 +944,13 @@ static ALWAYS_INLINE JSString* replaceAllWithCacheUsingRegExpSearchThreeArgument substring.getCharacters(buffer.subspan(bufferPos)); bufferPos += substring.length(); +#if USE(BUN_JSC_ADDITIONS) + auto replacement = asString(slot)->view(globalObject); + StringView { replacement }.getCharacters(buffer.subspan(bufferPos)); +#else auto replacement = asString(slot)->value(globalObject); StringView { replacement }.getCharacters(buffer.subspan(bufferPos)); +#endif bufferPos += replacement->length(); ++index; diff --git a/Source/JavaScriptCore/runtime/Structure.cpp b/Source/JavaScriptCore/runtime/Structure.cpp index bd9e01f437e4f..9f8f2b3e268dd 100644 --- a/Source/JavaScriptCore/runtime/Structure.cpp +++ b/Source/JavaScriptCore/runtime/Structure.cpp @@ -93,7 +93,11 @@ bool StructureTransitionTable::contains(PointerKey rep, unsigned attributes, Tra { if (isUsingSingleSlot()) { Structure* transition = trySingleTransition(); +#if USE(BUN_JSC_ADDITIONS) + return transition && transition->transitionPropertyName() == rep.pointer() && transition->transitionPropertyAttributes() == attributes && transition->transitionKind() == transitionKind; +#else return transition && transition->m_transitionPropertyName == rep.pointer() && transition->transitionPropertyAttributes() == attributes && transition->transitionKind() == transitionKind; +#endif } return map()->get(StructureTransitionTable::Hash::createKey(rep, attributes, transitionKind)); } @@ -484,6 +488,33 @@ PropertyTable* Structure::materializePropertyTable(VM& vm, bool setPropertyTable for (size_t i = structures.size(); i--;) { structure = structures[i]; +#if USE(BUN_JSC_ADDITIONS) + if (!structure->transitionPropertyName()) + continue; + switch (structure->transitionKind()) { + case TransitionKind::PropertyAddition: { + PropertyTableEntry entry(structure->transitionPropertyName(), structure->transitionOffset(), structure->transitionPropertyAttributes()); + auto nextOffset = table->nextOffset(structure->inlineCapacity()); + ASSERT_UNUSED(nextOffset, nextOffset == structure->transitionOffset()); + auto [offset, attribute, result] = table->add(vm, entry); + ASSERT_UNUSED(result, result); + ASSERT_UNUSED(offset, offset == nextOffset); + UNUSED_VARIABLE(attribute); + break; + } + case TransitionKind::PropertyDeletion: { + auto [offset, attributes] = table->take(vm, structure->transitionPropertyName()); + ASSERT_UNUSED(offset, offset != invalidOffset); + UNUSED_VARIABLE(attributes); + table->addDeletedOffset(structure->transitionOffset()); + break; + } + case TransitionKind::PropertyAttributeChange: { + PropertyOffset offset = table->updateAttributeIfExists(structure->transitionPropertyName(), structure->transitionPropertyAttributes()); + ASSERT_UNUSED(offset, offset == structure->transitionOffset()); + break; + } +#else if (!structure->m_transitionPropertyName) continue; switch (structure->transitionKind()) { @@ -509,6 +540,7 @@ PropertyTable* Structure::materializePropertyTable(VM& vm, bool setPropertyTable ASSERT_UNUSED(offset, offset == structure->transitionOffset()); break; } +#endif case TransitionKind::SetBrand: { continue; } @@ -601,7 +633,11 @@ Structure* Structure::addNewPropertyTransition(VM& vm, Structure* structure, Pro } transition->m_blob.setIndexingModeIncludingHistory(structure->indexingModeIncludingHistory() & ~CopyOnWrite); +#if USE(BUN_JSC_ADDITIONS) + transition->setTransitionPropertyName(propertyName.uid()); +#else transition->m_transitionPropertyName = propertyName.uid(); +#endif transition->setTransitionPropertyAttributes(attributes); transition->setTransitionKind(TransitionKind::PropertyAddition); transition->setPropertyTable(vm, structure->takePropertyTableOrCloneIfPinned(vm)); @@ -698,7 +734,11 @@ Structure* Structure::removeNewPropertyTransition(VM& vm, Structure* structure, } transition->m_blob.setIndexingModeIncludingHistory(structure->indexingModeIncludingHistory() & ~CopyOnWrite); +#if USE(BUN_JSC_ADDITIONS) + transition->setTransitionPropertyName(propertyName.uid()); +#else transition->m_transitionPropertyName = propertyName.uid(); +#endif transition->setTransitionKind(TransitionKind::PropertyDeletion); transition->setPropertyTable(vm, structure->takePropertyTableOrCloneIfPinned(vm)); transition->setMaxOffset(vm, structure->maxOffset()); @@ -839,7 +879,11 @@ Structure* Structure::attributeChangeTransition(VM& vm, Structure* structure, Pr } transition->m_blob.setIndexingModeIncludingHistory(structure->indexingModeIncludingHistory() & ~CopyOnWrite); +#if USE(BUN_JSC_ADDITIONS) + transition->setTransitionPropertyName(propertyName.uid()); +#else transition->m_transitionPropertyName = propertyName.uid(); +#endif transition->setTransitionPropertyAttributes(attributes); transition->setTransitionKind(TransitionKind::PropertyAttributeChange); transition->setPropertyTable(vm, structure->takePropertyTableOrCloneIfPinned(vm)); @@ -1114,7 +1158,11 @@ void Structure::pinForCaching(const AbstractLocker&, VM& vm, PropertyTable* tabl { setIsPinnedPropertyTable(true); setPropertyTable(vm, table); +#if USE(BUN_JSC_ADDITIONS) + clearTransitionPropertyName(); +#else m_transitionPropertyName = nullptr; +#endif } void Structure::allocateRareData(VM& vm) @@ -1233,6 +1281,32 @@ PropertyOffset Structure::getConcurrently(UniquedStringImpl* uid, unsigned& attr bool didFindStructure = findStructuresAndMapForMaterialization(structures, tableStructure, table); for (auto* structure : structures) { +#if USE(BUN_JSC_ADDITIONS) + if (!structure->transitionPropertyName()) + continue; + + switch (structure->transitionKind()) { + case TransitionKind::PropertyAddition: + case TransitionKind::PropertyAttributeChange: + break; + case TransitionKind::PropertyDeletion: + if (structure->transitionPropertyName() == uid) { + if (didFindStructure) { + assertIsHeld(tableStructure->m_lock); // Sadly Clang needs some help here. + tableStructure->m_lock.unlock(); + } + return invalidOffset; + } + continue; + case TransitionKind::SetBrand: + continue; + default: + ASSERT_NOT_REACHED(); + break; + } + + if (structure->transitionPropertyName() == uid) { +#else if (!structure->m_transitionPropertyName) continue; @@ -1257,6 +1331,7 @@ PropertyOffset Structure::getConcurrently(UniquedStringImpl* uid, unsigned& attr } if (structure->m_transitionPropertyName.get() == uid) { +#endif PropertyOffset result = structure->transitionOffset(); attributes = structure->transitionPropertyAttributes(); if (didFindStructure) { @@ -1344,8 +1419,8 @@ void Structure::getPropertyNamesFromStructure(VM& vm, PropertyNameArrayBuilder& table->forEachProperty([&](const auto& entry) { ASSERT(!isQuickPropertyAccessAllowedForEnumeration() || !(entry.attributes() & PropertyAttribute::DontEnum)); - ASSERT(!isQuickPropertyAccessAllowedForEnumeration() || !entry.key()->isSymbol()); - if (entry.key()->isSymbol()) { + ASSERT(!isQuickPropertyAccessAllowedForEnumeration() || !uidIsSymbol(entry.key())); + if (uidIsSymbol(entry.key())) { foundSymbol = true; if (propertyNames.propertyNameMode() != PropertyNameMode::Symbols) return IterationStatus::Continue; @@ -1358,7 +1433,7 @@ void Structure::getPropertyNamesFromStructure(VM& vm, PropertyNameArrayBuilder& // To ensure the order defined in the spec, we append symbols at the last elements of keys. // https://tc39.es/ecma262/#sec-ordinaryownpropertykeys table->forEachProperty([&](const auto& entry) { - if (entry.key()->isSymbol()) + if (uidIsSymbol(entry.key())) checkDontEnumAndAdd(entry); return IterationStatus::Continue; }); @@ -1718,7 +1793,11 @@ Structure* Structure::setBrandTransition(VM& vm, Structure* structure, Symbol* b transition->m_cachedPrototypeChain.setMayBeNull(vm, transition, structure->m_cachedPrototypeChain.get()); transition->m_blob.setIndexingModeIncludingHistory(structure->indexingModeIncludingHistory()); +#if USE(BUN_JSC_ADDITIONS) + transition->setTransitionPropertyName(&brand->uid()); +#else transition->m_transitionPropertyName = &brand->uid(); +#endif transition->setTransitionPropertyAttributes(0); transition->setPropertyTable(vm, structure->takePropertyTableOrCloneIfPinned(vm)); transition->setMaxOffset(vm, structure->maxOffset()); diff --git a/Source/JavaScriptCore/runtime/Structure.h b/Source/JavaScriptCore/runtime/Structure.h index de5071e126920..e67761fe6cf66 100644 --- a/Source/JavaScriptCore/runtime/Structure.h +++ b/Source/JavaScriptCore/runtime/Structure.h @@ -787,7 +787,17 @@ class Structure : public JSCell { static bool shouldConvertToPolyProto(const Structure* a, const Structure* b); +#if USE(BUN_JSC_ADDITIONS) + // Back to the upstream CompactRefPtr layout with fiber-aware ref/deref traits: + // RAII dtor handles the deref (so ~Structure stays = default and GC sweep is + // trivially destructible when enableIdentifierFiberWords is off), and operator= + // replaces the manual uidRef/uidDeref setter. + UniquedStringImpl* transitionPropertyName() const { return m_transitionPropertyName.get(); } + void setTransitionPropertyName(UniquedStringImpl* rep) { m_transitionPropertyName = rep; } + void clearTransitionPropertyName() { m_transitionPropertyName = nullptr; } +#else UniquedStringImpl* transitionPropertyName() const { return m_transitionPropertyName.get(); } +#endif struct PropertyHashEntry { const HashTable* table; @@ -1027,7 +1037,21 @@ class Structure : public JSCell { WriteBarrier m_previousOrRareData; +#if USE(BUN_JSC_ADDITIONS) + // CompactRefPtr storage + FiberAwareRefDerefTraits: same packing as upstream, + // fiber-safe when enableIdentifierFiberWords is on, identical to CompactRefPtr + // (tag-bit test is always false) when off. + RefPtr, FiberAwareRefDerefTraits> m_transitionPropertyName; + // A maxFiberWordKeyLength (5-char, 8-bit) fiber word occupies byte0 tag|is8Bit|len + // + bytes1..5 payload = 48 bits; CompactPtr on !HAVE(36BIT_ADDRESS) stores a raw + // uintptr_t so encode/decode is the identity and the word round-trips losslessly. + static_assert(!enableIdentifierFiberWords + || (!CompactPtrTraits::is32Bit + && (1 + maxFiberWordKeyLength) * 8 <= sizeof(CompactPtr::StorageType) * 8), + "CompactPtrTraits must round-trip a 48-bit fiber-word m_transitionPropertyName"); +#else CompactRefPtr m_transitionPropertyName; +#endif const ClassInfo* m_classInfo; diff --git a/Source/JavaScriptCore/runtime/StructureInlines.h b/Source/JavaScriptCore/runtime/StructureInlines.h index dafcbeaa73fd3..aab92199e9992 100644 --- a/Source/JavaScriptCore/runtime/StructureInlines.h +++ b/Source/JavaScriptCore/runtime/StructureInlines.h @@ -27,6 +27,9 @@ #include "BigIntPrototype.h" #include "BrandedStructure.h" +#if USE(BUN_JSC_ADDITIONS) +#include "InlinePropertyKey.h" +#endif #include "JSArrayBufferView.h" #include "JSGlobalObject.h" #include "JSObjectInlines.h" @@ -83,6 +86,26 @@ void Structure::forEachPropertyConcurrently(const Functor& functor) UncheckedKeyHashSet seenProperties; for (auto* structure : structures) { +#if USE(BUN_JSC_ADDITIONS) + if (!structure->transitionPropertyName() || seenProperties.contains(structure->transitionPropertyName())) + continue; + + seenProperties.add(structure->transitionPropertyName()); + + switch (structure->transitionKind()) { + case TransitionKind::PropertyAddition: + case TransitionKind::PropertyAttributeChange: + break; + case TransitionKind::PropertyDeletion: + case TransitionKind::SetBrand: + continue; + default: + ASSERT_NOT_REACHED(); + break; + } + + if (!functor(PropertyTableEntry(structure->transitionPropertyName(), structure->transitionOffset(), structure->transitionPropertyAttributes()))) { +#else if (!structure->m_transitionPropertyName || seenProperties.contains(structure->m_transitionPropertyName.get())) continue; @@ -101,6 +124,7 @@ void Structure::forEachPropertyConcurrently(const Functor& functor) } if (!functor(PropertyTableEntry(structure->m_transitionPropertyName.get(), structure->transitionOffset(), structure->transitionPropertyAttributes()))) { +#endif if (didFindStructure) { assertIsHeld(tableStructure->m_lock); // Sadly Clang needs some help here. tableStructure->m_lock.unlock(); @@ -271,7 +295,11 @@ inline PropertyOffset Structure::add(VM& vm, PropertyName propertyName, unsigned PropertyOffset newOffset = table->nextOffset(m_inlineCapacity); +#if USE(BUN_JSC_ADDITIONS) + m_propertyHash = m_propertyHash ^ uidHash(rep); +#else m_propertyHash = m_propertyHash ^ rep->existingSymbolAwareHash(); +#endif m_seenProperties.add(CompactPtr::encode(rep)); auto [offset, attribute, result] = table->add(vm, PropertyTableEntry(rep, newOffset, attributes)); @@ -429,7 +457,11 @@ ALWAYS_INLINE auto Structure::addOrReplacePropertyWithoutTransition(VM& vm, Prop PropertyOffset newOffset = table->nextOffset(m_inlineCapacity); +#if USE(BUN_JSC_ADDITIONS) + m_propertyHash = m_propertyHash ^ uidHash(rep); +#else m_propertyHash = m_propertyHash ^ rep->existingSymbolAwareHash(); +#endif m_seenProperties.add(CompactPtr::encode(rep)); auto [offset, attributes, result] = table->addAfterFind(vm, PropertyTableEntry(rep, newOffset, newAttributes), WTF::move(findResult)); @@ -481,7 +513,11 @@ inline void Structure::pin(const AbstractLocker&, VM& vm, PropertyTable* table) setIsPinnedPropertyTable(true); setPropertyTable(vm, table); clearPreviousID(); +#if USE(BUN_JSC_ADDITIONS) + clearTransitionPropertyName(); +#else m_transitionPropertyName = nullptr; +#endif } ALWAYS_INLINE bool Structure::shouldConvertToPolyProto(const Structure* a, const Structure* b) @@ -585,7 +621,11 @@ ALWAYS_INLINE StructureTransitionTable::Hash::Key StructureTransitionTable::Hash case TransitionKind::ChangePrototype: return StructureTransitionTable::Hash::createKey(structure->storedPrototype().isNull() ? nullptr : asObject(structure->storedPrototype()), structure->transitionPropertyAttributes(), structure->transitionKind()); default: +#if USE(BUN_JSC_ADDITIONS) + return StructureTransitionTable::Hash::createKey(structure->transitionPropertyName(), structure->transitionPropertyAttributes(), structure->transitionKind()); +#else return StructureTransitionTable::Hash::createKey(structure->m_transitionPropertyName.get(), structure->transitionPropertyAttributes(), structure->transitionKind()); +#endif } } diff --git a/Source/JavaScriptCore/runtime/StructureTransitionTable.h b/Source/JavaScriptCore/runtime/StructureTransitionTable.h index 44c577253e357..e081ec541fc0e 100644 --- a/Source/JavaScriptCore/runtime/StructureTransitionTable.h +++ b/Source/JavaScriptCore/runtime/StructureTransitionTable.h @@ -30,6 +30,9 @@ #include "WeakGCMap.h" #include #include +#if USE(BUN_JSC_ADDITIONS) +#include "InlinePropertyKey.h" +#endif namespace JSC { @@ -187,6 +190,11 @@ class StructureTransitionTable { static constexpr unsigned transitionKindShift = 56; static constexpr uintptr_t stringMask = (1ULL << attributesShift) - 1; static constexpr uintptr_t hashTableDeletedValue = 0x2; +#if USE(BUN_JSC_ADDITIONS) + // Fiber words share bit 1 with hashTableDeletedValue but always carry + // >=2 payload bytes, so the encoded word can never equal exactly 0x2. + static_assert(JSC::inlinePropertyKeyTag == hashTableDeletedValue, ""); +#endif static_assert(sizeof(TransitionPropertyAttributes) * 8 <= 8); static_assert(sizeof(TransitionKind) * 8 <= 8); static_assert(hashTableDeletedValue < 8); @@ -197,7 +205,11 @@ class StructureTransitionTable { : m_encodedData(impl.raw() | (static_cast(attributes) << attributesShift) | (static_cast(transitionKind) << transitionKindShift)) { ASSERT(impl == this->impl()); +#if USE(BUN_JSC_ADDITIONS) + ASSERT(isInlinePropertyKey(impl.raw()) || roundUpToMultipleOf<8>(impl.raw()) == impl.raw()); +#else ASSERT(roundUpToMultipleOf<8>(impl.raw()) == impl.raw()); +#endif ASSERT(attributes <= UINT8_MAX); ASSERT(attributes == this->attributes()); ASSERT(transitionKind != TransitionKind::Unknown); diff --git a/Source/JavaScriptCore/runtime/SymbolTable.h b/Source/JavaScriptCore/runtime/SymbolTable.h index 935cc35109744..f35ef1cac27c7 100644 --- a/Source/JavaScriptCore/runtime/SymbolTable.h +++ b/Source/JavaScriptCore/runtime/SymbolTable.h @@ -434,10 +434,17 @@ class SymbolTable final : public JSCell { typedef JSCell Base; static constexpr unsigned StructureFlags = Base::StructureFlags | StructureIsImmortal; +#if USE(BUN_JSC_ADDITIONS) + typedef UncheckedKeyHashMap, SymbolTableIndexHashTraits> Map; + typedef UncheckedKeyHashMap UniqueIDMap; + typedef UncheckedKeyHashMap, IdentifierRepHash> UniqueTypeSetMap; + typedef UncheckedKeyHashMap OffsetToVariableMap; +#else typedef UncheckedKeyHashMap, SymbolTableEntry, IdentifierRepHash, HashTraits>, SymbolTableIndexHashTraits> Map; typedef UncheckedKeyHashMap, GlobalVariableID, IdentifierRepHash> UniqueIDMap; typedef UncheckedKeyHashMap, RefPtr, IdentifierRepHash> UniqueTypeSetMap; typedef UncheckedKeyHashMap> OffsetToVariableMap; +#endif typedef Vector LocalToEntryVec; typedef WTF::IteratorRange PrivateNameIteratorRange; @@ -583,6 +590,15 @@ class SymbolTable final : public JSCell { return makeIteratorRange(rareData.m_privateNames.begin(), rareData.m_privateNames.end()); } +#if USE(BUN_JSC_ADDITIONS) + void addPrivateName(UniquedStringImpl* key, PrivateNameEntry value) + { + ASSERT(key && !uidIsSymbol(key)); + auto& rareData = ensureRareData(); + ASSERT(rareData.m_privateNames.find(key) == rareData.m_privateNames.end()); + rareData.m_privateNames.add(FiberAwareRefPtr(key), value); + } +#else void addPrivateName(const RefPtr& key, PrivateNameEntry value) { ASSERT(key && !key->isSymbol()); @@ -590,13 +606,23 @@ class SymbolTable final : public JSCell { ASSERT(rareData.m_privateNames.find(key) == rareData.m_privateNames.end()); rareData.m_privateNames.add(key, value); } +#endif +#if USE(BUN_JSC_ADDITIONS) + bool hasPrivateName(const FiberAwareRefPtr& key) const + { + if (auto* rareData = m_rareData.get()) + return rareData->m_privateNames.contains(key); + return false; + } +#else bool hasPrivateName(const RefPtr& key) const { if (auto* rareData = m_rareData.get()) return rareData->m_privateNames.contains(key); return false; } +#endif template void set(const ConcurrentJSLocker&, UniquedStringImpl* key, Entry&& entry) diff --git a/Source/JavaScriptCore/runtime/VM.h b/Source/JavaScriptCore/runtime/VM.h index 6df90ecf4b5b8..40181cb8f112b 100644 --- a/Source/JavaScriptCore/runtime/VM.h +++ b/Source/JavaScriptCore/runtime/VM.h @@ -42,6 +42,9 @@ WTF_ALLOW_UNSAFE_BUFFER_USAGE_BEGIN #include "JSDateMath.h" #include "JSONAtomStringCache.h" #include "KeyAtomStringCache.h" +#if USE(BUN_JSC_ADDITIONS) +#include "InlineStringCache.h" +#endif #include "NativeFunction.h" #include "NumericStrings.h" #include "SmallStrings.h" @@ -633,6 +636,10 @@ class VM : public ThreadSafeRefCountedWithSuppressingSaferCPPChecking { Ref lastAtomizedIdentifierAtomStringImpl { *static_cast(StringImpl::empty()) }; JSONAtomStringCache jsonAtomStringCache; KeyAtomStringCache keyAtomStringCache; +#if USE(BUN_JSC_ADDITIONS) + InlineStringCache inlineStringCache; + InlineAtomCache inlineAtomCache; +#endif StringSplitCache stringSplitCache; Vector stringSplitIndice; StringReplaceCache stringReplaceCache; diff --git a/Source/JavaScriptCore/wasm/js/JSWebAssemblyModule.cpp b/Source/JavaScriptCore/wasm/js/JSWebAssemblyModule.cpp index b583635ea763d..1c178d3e2a618 100644 --- a/Source/JavaScriptCore/wasm/js/JSWebAssemblyModule.cpp +++ b/Source/JavaScriptCore/wasm/js/JSWebAssemblyModule.cpp @@ -76,7 +76,15 @@ void JSWebAssemblyModule::finishCreation(VM& vm) } for (auto& exp : moduleInformation.exports) { auto offset = exportSymbolTable->takeNextScopeOffset(NoLockingNecessary); +#if USE(BUN_JSC_ADDITIONS) + // D.4 coherence: WebAssemblyModuleRecord::initializeExports() looks these + // up via Identifier::fromString() which yields a fiber word for 2..5-char + // names; key the table with the same canonical representation so the + // pointer-identity compare in SymbolTable::Map hits. + exportSymbolTable->set(NoLockingNecessary, Identifier::fromString(vm, makeAtomString(exp.field)).impl(), SymbolTableEntry(VarOffset(offset))); +#else exportSymbolTable->set(NoLockingNecessary, makeAtomString(exp.field).impl(), SymbolTableEntry(VarOffset(offset))); +#endif } m_exportSymbolTable.set(vm, this, exportSymbolTable);