From b85f1c61efdbbf7413de754290959cc4236e3523 Mon Sep 17 00:00:00 2001 From: Andy Ayers Date: Sun, 5 Dec 2021 14:58:53 -0800 Subject: [PATCH] Support for devirtualizing array interface methods Update jit and runtime to devirtualize interface calls on arrays over non-shared element types. Shared types are not handled. The resulting enumerator calls would not inline as they are on a different class. This addresses the first two issues noted in #62457. --- src/coreclr/jit/importer.cpp | 512 +++++++++++++++++++++++++++++++- src/coreclr/vm/jitinterface.cpp | 44 ++- 2 files changed, 543 insertions(+), 13 deletions(-) diff --git a/src/coreclr/jit/importer.cpp b/src/coreclr/jit/importer.cpp index 9dc29e2b989591..56c68ba39b6af9 100644 --- a/src/coreclr/jit/importer.cpp +++ b/src/coreclr/jit/importer.cpp @@ -13902,18 +13902,510 @@ bool Compiler::impInlineIsGuaranteedThisDerefBeforeAnySideEffects(GenTree* ad } //------------------------------------------------------------------------ -// impAllocateMethodPointerInfo: create methodPointerInfo into jit-allocated memory and init it. +// SpillRetExprHelper: iterate through arguments tree and spill ret_expr to local variables. +// +class SpillRetExprHelper +{ +public: + SpillRetExprHelper(Compiler* comp) : comp(comp) + { + } + + void StoreRetExprResultsInArgs(GenTreeCall* call) + { + for (CallArg& arg : call->gtArgs.Args()) + { + comp->fgWalkTreePre(&arg.EarlyNodeRef(), SpillRetExprVisitor, this); + } + } + +private: + static Compiler::fgWalkResult SpillRetExprVisitor(GenTree** pTree, Compiler::fgWalkData* fgWalkPre) + { + assert((pTree != nullptr) && (*pTree != nullptr)); + GenTree* tree = *pTree; + if ((tree->gtFlags & GTF_CALL) == 0) + { + // Trees with ret_expr are marked as GTF_CALL. + return Compiler::WALK_SKIP_SUBTREES; + } + if (tree->OperGet() == GT_RET_EXPR) + { + SpillRetExprHelper* walker = static_cast(fgWalkPre->pCallbackData); + walker->StoreRetExprAsLocalVar(pTree); + } + return Compiler::WALK_CONTINUE; + } + + void StoreRetExprAsLocalVar(GenTree** pRetExpr) + { + GenTree* retExpr = *pRetExpr; + assert(retExpr->OperGet() == GT_RET_EXPR); + const unsigned tmp = comp->lvaGrabTemp(true DEBUGARG("spilling ret_expr")); + JITDUMP("Storing return expression [%06u] to a local var V%02u.\n", comp->dspTreeID(retExpr), tmp); + comp->impAssignTempGen(tmp, retExpr, (unsigned)Compiler::CHECK_SPILL_NONE); + *pRetExpr = comp->gtNewLclvNode(tmp, retExpr->TypeGet()); + + if (retExpr->TypeGet() == TYP_REF) + { + assert(comp->lvaTable[tmp].lvSingleDef == 0); + comp->lvaTable[tmp].lvSingleDef = 1; + JITDUMP("Marked V%02u as a single def temp\n", tmp); + + bool isExact = false; + bool isNonNull = false; + CORINFO_CLASS_HANDLE retClsHnd = comp->gtGetClassHandle(retExpr, &isExact, &isNonNull); + if (retClsHnd != nullptr) + { + comp->lvaSetClass(tmp, retClsHnd, isExact); + } + } + } + +private: + Compiler* comp; +}; + +//------------------------------------------------------------------------ +// addFatPointerCandidate: mark the call and the method, that they have a fat pointer candidate. +// Spill ret_expr in the call node, because they can't be cloned. // // Arguments: -// token - init value for the allocated token. -// tokenConstrained - init value for the constraint associated with the token +// call - fat calli candidate // -// Return Value: -// pointer to token into jit-allocated memory. -methodPointerInfo* Compiler::impAllocateMethodPointerInfo(const CORINFO_RESOLVED_TOKEN& token, mdToken tokenConstrained) +void Compiler::addFatPointerCandidate(GenTreeCall* call) { - methodPointerInfo* memory = getAllocator(CMK_Unknown).allocate(1); - memory->m_token = token; - memory->m_tokenConstraint = tokenConstrained; - return memory; + JITDUMP("Marking call [%06u] as fat pointer candidate\n", dspTreeID(call)); + setMethodHasFatPointer(); + call->SetFatPointerCandidate(); + SpillRetExprHelper helper(this); + helper.StoreRetExprResultsInArgs(call); +} + +//------------------------------------------------------------------------ +// considerGuardedDevirtualization: see if we can profitably guess at the +// class involved in an interface or virtual call. +// +// Arguments: +// +// call - potential guarded devirtualization candidate +// ilOffset - IL ofset of the call instruction +// isInterface - true if this is an interface call +// baseMethod - target method of the call +// baseClass - class that introduced the target method +// pContextHandle - context handle for the call +// objClass - class of 'this' in the call +// objClassName - name of the obj Class +// +// Notes: +// Consults with VM to see if there's a likely class at runtime, +// if so, adds a candidate for guarded devirtualization. +// +void Compiler::considerGuardedDevirtualization( + GenTreeCall* call, + IL_OFFSET ilOffset, + bool isInterface, + CORINFO_METHOD_HANDLE baseMethod, + CORINFO_CLASS_HANDLE baseClass, + CORINFO_CONTEXT_HANDLE* pContextHandle DEBUGARG(CORINFO_CLASS_HANDLE objClass) DEBUGARG(const char* objClassName)) +{ +#if defined(DEBUG) + const char* callKind = isInterface ? "interface" : "virtual"; +#endif + + JITDUMP("Considering guarded devirtualization at IL offset %u (0x%x)\n", ilOffset, ilOffset); + + // We currently only get likely class guesses when there is PGO data + // with class profiles. + // + if (fgPgoClassProfiles == 0) + { + JITDUMP("Not guessing for class: no class profile pgo data, or pgo disabled\n"); + return; + } + + // See if there's a likely guess for the class. + // + const unsigned likelihoodThreshold = isInterface ? 25 : 30; + unsigned likelihood = 0; + unsigned numberOfClasses = 0; + + CORINFO_CLASS_HANDLE likelyClass = NO_CLASS_HANDLE; + + bool doRandomDevirt = false; + + const int maxLikelyClasses = 32; + LikelyClassRecord likelyClasses[maxLikelyClasses]; + +#ifdef DEBUG + // Optional stress mode to pick a random known class, rather than + // the most likely known class. + // + doRandomDevirt = JitConfig.JitRandomGuardedDevirtualization() != 0; + + if (doRandomDevirt) + { + // Reuse the random inliner's random state. + // + CLRRandom* const random = + impInlineRoot()->m_inlineStrategy->GetRandom(JitConfig.JitRandomGuardedDevirtualization()); + likelyClasses[0].clsHandle = getRandomClass(fgPgoSchema, fgPgoSchemaCount, fgPgoData, ilOffset, random); + likelyClasses[0].likelihood = 100; + if (likelyClasses[0].clsHandle != NO_CLASS_HANDLE) + { + numberOfClasses = 1; + } + } + else +#endif + { + numberOfClasses = + getLikelyClasses(likelyClasses, maxLikelyClasses, fgPgoSchema, fgPgoSchemaCount, fgPgoData, ilOffset); + } + + // For now we only use the most popular type + + likelihood = likelyClasses[0].likelihood; + likelyClass = likelyClasses[0].clsHandle; + + if (numberOfClasses < 1) + { + JITDUMP("No likely class, sorry\n"); + return; + } + + assert(likelyClass != NO_CLASS_HANDLE); + + // Print all likely classes + JITDUMP("%s classes for %p (%s):\n", doRandomDevirt ? "Random" : "Likely", dspPtr(objClass), objClassName) + for (UINT32 i = 0; i < numberOfClasses; i++) + { + JITDUMP(" %u) %p (%s) [likelihood:%u%%]\n", i + 1, likelyClasses[i].clsHandle, + eeGetClassName(likelyClasses[i].clsHandle), likelyClasses[i].likelihood); + } + + // Todo: a more advanced heuristic using likelihood, number of + // classes, and the profile count for this block. + // + // For now we will guess if the likelihood is at least 25%/30% (intfc/virt), as studies + // have shown this transformation should pay off even if we guess wrong sometimes. + // + if (likelihood < likelihoodThreshold) + { + JITDUMP("Not guessing for class; likelihood is below %s call threshold %u\n", callKind, likelihoodThreshold); + return; + } + + uint32_t const likelyClassAttribs = info.compCompHnd->getClassAttribs(likelyClass); + + if ((likelyClassAttribs & CORINFO_FLG_ABSTRACT) != 0) + { + // We may see an abstract likely class, if we have a stale profile. + // No point guessing for this. + // + JITDUMP("Not guessing for class; abstract (stale profile)\n"); + return; + } + + // Figure out which method will be called. + // + CORINFO_DEVIRTUALIZATION_INFO dvInfo; + dvInfo.virtualMethod = baseMethod; + dvInfo.objClass = likelyClass; + dvInfo.context = *pContextHandle; + dvInfo.exactContext = *pContextHandle; + dvInfo.pResolvedTokenVirtualMethod = nullptr; + + const bool canResolve = info.compCompHnd->resolveVirtualMethod(&dvInfo); + + if (!canResolve) + { + JITDUMP("Can't figure out which method would be invoked, sorry (reason %d)\n", dvInfo.detail); + return; + } + + CORINFO_METHOD_HANDLE likelyMethod = dvInfo.devirtualizedMethod; + JITDUMP("%s call would invoke method %s\n", callKind, eeGetMethodName(likelyMethod, nullptr)); + + // Add this as a potential candidate. + // + uint32_t const likelyMethodAttribs = info.compCompHnd->getMethodAttribs(likelyMethod); + addGuardedDevirtualizationCandidate(call, likelyMethod, likelyClass, likelyMethodAttribs, likelyClassAttribs, + likelihood); +} + +//------------------------------------------------------------------------ +// addGuardedDevirtualizationCandidate: potentially mark the call as a guarded +// devirtualization candidate +// +// Notes: +// +// Call sites in rare or unoptimized code, and calls that require cookies are +// not marked as candidates. +// +// As part of marking the candidate, the code spills GT_RET_EXPRs anywhere in any +// child tree, because and we need to clone all these trees when we clone the call +// as part of guarded devirtualization, and these IR nodes can't be cloned. +// +// Arguments: +// call - potential guarded devirtualization candidate +// methodHandle - method that will be invoked if the class test succeeds +// classHandle - class that will be tested for at runtime +// methodAttr - attributes of the method +// classAttr - attributes of the class +// likelihood - odds that this class is the class seen at runtime +// +void Compiler::addGuardedDevirtualizationCandidate(GenTreeCall* call, + CORINFO_METHOD_HANDLE methodHandle, + CORINFO_CLASS_HANDLE classHandle, + unsigned methodAttr, + unsigned classAttr, + unsigned likelihood) +{ + // This transformation only makes sense for virtual calls + assert(call->IsVirtual()); + + // Only mark calls if the feature is enabled. + const bool isEnabled = JitConfig.JitEnableGuardedDevirtualization() > 0; + + if (!isEnabled) + { + JITDUMP("NOT Marking call [%06u] as guarded devirtualization candidate -- disabled by jit config\n", + dspTreeID(call)); + return; + } + + // Bail if not optimizing or the call site is very likely cold + if (compCurBB->isRunRarely() || opts.OptimizationDisabled()) + { + JITDUMP("NOT Marking call [%06u] as guarded devirtualization candidate -- rare / dbg / minopts\n", + dspTreeID(call)); + return; + } + + // CT_INDIRECT calls may use the cookie, bail if so... + // + // If transforming these provides a benefit, we could save this off in the same way + // we save the stub address below. + if ((call->gtCallType == CT_INDIRECT) && (call->AsCall()->gtCallCookie != nullptr)) + { + JITDUMP("NOT Marking call [%06u] as guarded devirtualization candidate -- CT_INDIRECT with cookie\n", + dspTreeID(call)); + return; + } + +#ifdef DEBUG + + // See if disabled by range + // + static ConfigMethodRange JitGuardedDevirtualizationRange; + JitGuardedDevirtualizationRange.EnsureInit(JitConfig.JitGuardedDevirtualizationRange()); + assert(!JitGuardedDevirtualizationRange.Error()); + if (!JitGuardedDevirtualizationRange.Contains(impInlineRoot()->info.compMethodHash())) + { + JITDUMP("NOT Marking call [%06u] as guarded devirtualization candidate -- excluded by " + "JitGuardedDevirtualizationRange", + dspTreeID(call)); + return; + } + +#endif + + // We're all set, proceed with candidate creation. + // + JITDUMP("Marking call [%06u] as guarded devirtualization candidate; will guess for class %s\n", dspTreeID(call), + eeGetClassName(classHandle)); + setMethodHasGuardedDevirtualization(); + call->SetGuardedDevirtualizationCandidate(); + + // Spill off any GT_RET_EXPR subtrees so we can clone the call. + // + SpillRetExprHelper helper(this); + helper.StoreRetExprResultsInArgs(call); + + // Gather some information for later. Note we actually allocate InlineCandidateInfo + // here, as the devirtualized half of this call will likely become an inline candidate. + // + GuardedDevirtualizationCandidateInfo* pInfo = new (this, CMK_Inlining) InlineCandidateInfo; + + pInfo->guardedMethodHandle = methodHandle; + pInfo->guardedMethodUnboxedEntryHandle = nullptr; + pInfo->guardedClassHandle = classHandle; + pInfo->likelihood = likelihood; + pInfo->requiresInstMethodTableArg = false; + + // If the guarded class is a value class, look for an unboxed entry point. + // + if ((classAttr & CORINFO_FLG_VALUECLASS) != 0) + { + JITDUMP(" ... class is a value class, looking for unboxed entry\n"); + bool requiresInstMethodTableArg = false; + CORINFO_METHOD_HANDLE unboxedEntryMethodHandle = + info.compCompHnd->getUnboxedEntry(methodHandle, &requiresInstMethodTableArg); + + if (unboxedEntryMethodHandle != nullptr) + { + JITDUMP(" ... updating GDV candidate with unboxed entry info\n"); + pInfo->guardedMethodUnboxedEntryHandle = unboxedEntryMethodHandle; + pInfo->requiresInstMethodTableArg = requiresInstMethodTableArg; + } + } + + call->gtGuardedDevirtualizationCandidateInfo = pInfo; +} + +void Compiler::addExpRuntimeLookupCandidate(GenTreeCall* call) +{ + setMethodHasExpRuntimeLookup(); + call->SetExpRuntimeLookup(); +} + +//------------------------------------------------------------------------ +// impIsClassExact: check if a class handle can only describe values +// of exactly one class. +// +// Arguments: +// classHnd - handle for class in question +// +// Returns: +// true if class is final and not subject to special casting from +// variance or similar. +// +// Note: +// We are conservative on arrays of primitive types here. + +bool Compiler::impIsClassExact(CORINFO_CLASS_HANDLE classHnd) +{ + DWORD flags = info.compCompHnd->getClassAttribs(classHnd); + DWORD flagsMask = CORINFO_FLG_FINAL | CORINFO_FLG_VARIANCE | CORINFO_FLG_ARRAY; + + if ((flags & flagsMask) == CORINFO_FLG_FINAL) + { + return true; + } + if ((flags & flagsMask) == (CORINFO_FLG_FINAL | CORINFO_FLG_ARRAY)) + { + CORINFO_CLASS_HANDLE arrayElementHandle = nullptr; + CorInfoType type = info.compCompHnd->getChildType(classHnd, &arrayElementHandle); + + if ((type == CORINFO_TYPE_CLASS) || (type == CORINFO_TYPE_VALUECLASS)) + { + return impIsClassExact(arrayElementHandle); + } + } + return false; +} + +//------------------------------------------------------------------------ +// impCanSkipCovariantStoreCheck: see if storing a ref type value to an array +// can skip the array store covariance check. +// +// Arguments: +// value -- tree producing the value to store +// array -- tree representing the array to store to +// +// Returns: +// true if the store does not require a covariance check. +// +bool Compiler::impCanSkipCovariantStoreCheck(GenTree* value, GenTree* array) +{ + // We should only call this when optimizing. + assert(opts.OptimizationEnabled()); + + // Check for assignment to same array, ie. arrLcl[i] = arrLcl[j] + if (value->OperIs(GT_INDEX) && array->OperIs(GT_LCL_VAR)) + { + GenTree* valueIndex = value->AsIndex()->Arr(); + if (valueIndex->OperIs(GT_LCL_VAR)) + { + unsigned valueLcl = valueIndex->AsLclVar()->GetLclNum(); + unsigned arrayLcl = array->AsLclVar()->GetLclNum(); + if ((valueLcl == arrayLcl) && !lvaGetDesc(arrayLcl)->IsAddressExposed()) + { + JITDUMP("\nstelem of ref from same array: skipping covariant store check\n"); + return true; + } + } + } + + // Check for assignment of NULL. + if (value->OperIs(GT_CNS_INT)) + { + assert(value->gtType == TYP_REF); + if (value->AsIntCon()->gtIconVal == 0) + { + JITDUMP("\nstelem of null: skipping covariant store check\n"); + return true; + } + // Non-0 const refs can only occur with frozen objects + assert(value->IsIconHandle(GTF_ICON_STR_HDL)); + assert(doesMethodHaveFrozenString() || + (compIsForInlining() && impInlineInfo->InlinerCompiler->doesMethodHaveFrozenString())); + } + + // Try and get a class handle for the array + if (value->gtType != TYP_REF) + { + return false; + } + + bool arrayIsExact = false; + bool arrayIsNonNull = false; + CORINFO_CLASS_HANDLE arrayHandle = gtGetClassHandle(array, &arrayIsExact, &arrayIsNonNull); + + if (arrayHandle == NO_CLASS_HANDLE) + { + return false; + } + + // There are some methods in corelib where we're storing to an array but the IL + // doesn't reflect this (see SZArrayHelper). Avoid. + DWORD attribs = info.compCompHnd->getClassAttribs(arrayHandle); + if ((attribs & CORINFO_FLG_ARRAY) == 0) + { + return false; + } + + CORINFO_CLASS_HANDLE arrayElementHandle = nullptr; + CorInfoType arrayElemType = info.compCompHnd->getChildType(arrayHandle, &arrayElementHandle); + + // Verify array type handle is really an array of ref type + assert(arrayElemType == CORINFO_TYPE_CLASS); + + // Check for exactly object[] + if (arrayIsExact && (arrayElementHandle == impGetObjectClass())) + { + JITDUMP("\nstelem to (exact) object[]: skipping covariant store check\n"); + return true; + } + + const bool arrayTypeIsSealed = impIsClassExact(arrayElementHandle); + + if ((!arrayIsExact && !arrayTypeIsSealed) || (arrayElementHandle == NO_CLASS_HANDLE)) + { + // Bail out if we don't know array's exact type + return false; + } + + bool valueIsExact = false; + bool valueIsNonNull = false; + CORINFO_CLASS_HANDLE valueHandle = gtGetClassHandle(value, &valueIsExact, &valueIsNonNull); + + // Array's type is sealed and equals to value's type + if (arrayTypeIsSealed && (valueHandle == arrayElementHandle)) + { + JITDUMP("\nstelem to T[] with T exact: skipping covariant store check\n"); + return true; + } + + // Array's type is not sealed but we know its exact type + if (arrayIsExact && (valueHandle != NO_CLASS_HANDLE) && + (info.compCompHnd->compareTypesForCast(valueHandle, arrayElementHandle) == TypeCompareState::Must)) + { + JITDUMP("\nstelem to T[] with T exact: skipping covariant store check\n"); + return true; + } + + return false; } +>>>>>>> f90bd197a22 (Support for devirtualizing array interface methods) diff --git a/src/coreclr/vm/jitinterface.cpp b/src/coreclr/vm/jitinterface.cpp index 9e265926481706..eb5f2c758e8feb 100644 --- a/src/coreclr/vm/jitinterface.cpp +++ b/src/coreclr/vm/jitinterface.cpp @@ -8609,7 +8609,27 @@ bool CEEInfo::resolveVirtualMethodHelper(CORINFO_DEVIRTUALIZATION_INFO * info) // // We must ensure that pObjMT actually implements the // interface corresponding to pBaseMD. - if (!pObjMT->CanCastToInterface(pBaseMT)) + // + if (pObjMT->IsArray()) + { + // If we're in a shared context we'll devirt to a shared + // generic method and won't be able to inline, so just bail. + // + if (pBaseMT->IsSharedByGenericInstantiations()) + { + info->detail = CORINFO_DEVIRTUALIZATION_FAILED_CANON; + return false; + } + + // Ensure we can cast the array to the interface type + // + if (!TypeHandle(pObjMT).CanCastTo(TypeHandle(pBaseMT))) + { + info->detail = CORINFO_DEVIRTUALIZATION_FAILED_CAST; + return false; + } + } + else if (!pObjMT->CanCastToInterface(pBaseMT)) { info->detail = CORINFO_DEVIRTUALIZATION_FAILED_CAST; return false; @@ -8719,8 +8739,12 @@ bool CEEInfo::resolveVirtualMethodHelper(CORINFO_DEVIRTUALIZATION_INFO * info) // We may fail to get an exact context if the method is a default // interface method. If so, we'll use the method's class. // + // For array -> interface devirt, the devirtualized methods + // will be defined by SZArrayHelper. + // MethodTable* pApproxMT = pDevirtMD->GetMethodTable(); MethodTable* pExactMT = pApproxMT; + bool isArray = false; if (pApproxMT->IsInterface()) { @@ -8729,6 +8753,10 @@ bool CEEInfo::resolveVirtualMethodHelper(CORINFO_DEVIRTUALIZATION_INFO * info) _ASSERTE(!pDevirtMD->HasClassInstantiation()); } + else if (pBaseMT->IsInterface() && pObjMT->IsArray()) + { + isArray = true; + } else { pExactMT = pDevirtMD->GetExactDeclaringType(pObjMT); @@ -8737,8 +8765,18 @@ bool CEEInfo::resolveVirtualMethodHelper(CORINFO_DEVIRTUALIZATION_INFO * info) // Success! Pass back the results. // info->devirtualizedMethod = (CORINFO_METHOD_HANDLE) pDevirtMD; - info->exactContext = MAKE_CLASSCONTEXT((CORINFO_CLASS_HANDLE) pExactMT); - info->requiresInstMethodTableArg = false; + + if (isArray) + { + info->exactContext = MAKE_METHODCONTEXT((CORINFO_METHOD_HANDLE) pDevirtMD); + info->requiresInstMethodTableArg = pDevirtMD->RequiresInstMethodTableArg(); + } + else + { + info->exactContext = MAKE_CLASSCONTEXT((CORINFO_CLASS_HANDLE) pExactMT); + info->requiresInstMethodTableArg = false; + } + info->detail = CORINFO_DEVIRTUALIZATION_SUCCESS; return true;