diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index 0ab30837412ff7..193b6ac120df0f 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -1,32246 +1,14495 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -/*XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX -XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX -XX XX -XX GenTree XX -XX XX -XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX -XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX -*/ - -#include "jitpch.h" -#include "hwintrinsic.h" -#include "simd.h" - -#ifdef _MSC_VER -#pragma hdrstop -#endif - -/*****************************************************************************/ - -const unsigned char GenTree::gtOperKindTable[] = { -#define GTNODE(en, st, cm, ivn, ok) ((ok) & GTK_MASK) + GTK_COMMUTE *cm, -#include "gtlist.h" -}; - -#ifdef DEBUG -const GenTreeDebugOperKind GenTree::gtDebugOperKindTable[] = { -#define GTNODE(en, st, cm, ivn, ok) static_cast((ok) & DBK_MASK), -#include "gtlist.h" -}; -#endif // DEBUG - -/***************************************************************************** - * - * The types of different GenTree nodes - */ - -#ifdef DEBUG - -#define INDENT_SIZE 3 - -//-------------------------------------------- -// -// IndentStack: This struct is used, along with its related enums and strings, -// to control both the indendtation and the printing of arcs. -// -// Notes: -// The mode of printing is set in the Constructor, using its 'compiler' argument. -// Currently it only prints arcs when fgOrder == fgOrderLinear. -// The type of arc to print is specified by the IndentInfo enum, and is controlled -// by the caller of the Push() method. - -enum IndentChars -{ - ICVertical, - ICBottom, - ICTop, - ICMiddle, - ICDash, - ICTerminal, - ICError, - IndentCharCount -}; - -// clang-format off -// Sets of strings for different dumping options vert bot top mid dash embedded terminal error -static const char* emptyIndents[IndentCharCount] = { " ", " ", " ", " ", " ", "", "?" }; -static const char* asciiIndents[IndentCharCount] = { "|", "\\", "/", "+", "-", "*", "?" }; -static const char* unicodeIndents[IndentCharCount] = { "\xe2\x94\x82", "\xe2\x94\x94", "\xe2\x94\x8c", "\xe2\x94\x9c", "\xe2\x94\x80", "\xe2\x96\x8c", "?" }; -// clang-format on - -typedef ArrayStack IndentInfoStack; -struct IndentStack -{ - IndentInfoStack stack; - const char** indents; - - // Constructor for IndentStack. Uses 'compiler' to determine the mode of printing. - IndentStack(Compiler* compiler) - : stack(compiler->getAllocator(CMK_DebugOnly)) - { - if (compiler->asciiTrees) - { - indents = asciiIndents; - } - else - { - indents = unicodeIndents; - } - } - - // Return the depth of the current indentation. - unsigned Depth() - { - return stack.Height(); - } - - // Push a new indentation onto the stack, of the given type. - void Push(Compiler::IndentInfo info) - { - stack.Push(info); - } - - // Pop the most recent indentation type off the stack. - Compiler::IndentInfo Pop() - { - return stack.Pop(); - } - - // Print the current indentation and arcs. - void print() - { - unsigned indentCount = Depth(); - for (unsigned i = 0; i < indentCount; i++) - { - unsigned index = indentCount - 1 - i; - switch (stack.Top(index)) - { - case Compiler::IndentInfo::IINone: - printf(" "); - break; - case Compiler::IndentInfo::IIArc: - if (index == 0) - { - printf("%s%s%s", indents[ICMiddle], indents[ICDash], indents[ICDash]); - } - else - { - printf("%s ", indents[ICVertical]); - } - break; - case Compiler::IndentInfo::IIArcBottom: - printf("%s%s%s", indents[ICBottom], indents[ICDash], indents[ICDash]); - break; - case Compiler::IndentInfo::IIArcTop: - printf("%s%s%s", indents[ICTop], indents[ICDash], indents[ICDash]); - break; - case Compiler::IndentInfo::IIError: - printf("%s%s%s", indents[ICError], indents[ICDash], indents[ICDash]); - break; - default: - unreached(); - } - } - printf("%s", indents[ICTerminal]); - } -}; - -//------------------------------------------------------------------------ -// printIndent: This is a static method which simply invokes the 'print' -// method on its 'indentStack' argument. -// -// Arguments: -// indentStack - specifies the information for the indentation & arcs to be printed -// -// Notes: -// This method exists to localize the checking for the case where indentStack is null. - -static void printIndent(IndentStack* indentStack) -{ - if (indentStack == nullptr) - { - return; - } - indentStack->print(); -} - -#endif - -#if defined(DEBUG) || CALL_ARG_STATS || COUNT_BASIC_BLOCKS || COUNT_LOOPS || EMITTER_STATS || MEASURE_MEM_ALLOC || \ - NODEBASH_STATS || MEASURE_NODE_SIZE || COUNT_AST_OPERS || DUMP_FLOWGRAPHS - -static const char* opNames[] = { -#define GTNODE(en, st, cm, ivn, ok) #en, -#include "gtlist.h" -}; - -const char* GenTree::OpName(genTreeOps op) -{ - assert((unsigned)op < ArrLen(opNames)); - - return opNames[op]; -} - -#endif - -#if MEASURE_NODE_SIZE - -static const char* opStructNames[] = { -#define GTNODE(en, st, cm, ivn, ok) #st, -#include "gtlist.h" -}; - -const char* GenTree::OpStructName(genTreeOps op) -{ - assert((unsigned)op < ArrLen(opStructNames)); - - return opStructNames[op]; -} - -#endif - -// -// We allocate tree nodes in 2 different sizes: -// - TREE_NODE_SZ_SMALL for most nodes -// - TREE_NODE_SZ_LARGE for the few nodes (such as calls) that have -// more fields and take up a lot more space. -// - -/* GT_COUNT'th oper is overloaded as 'undefined oper', so allocate storage for GT_COUNT'th oper also */ -/* static */ -unsigned char GenTree::s_gtNodeSizes[GT_COUNT + 1]; - -#if NODEBASH_STATS || MEASURE_NODE_SIZE || COUNT_AST_OPERS - -unsigned char GenTree::s_gtTrueSizes[GT_COUNT + 1]{ -#define GTNODE(en, st, cm, ivn, ok) sizeof(st), -#include "gtlist.h" -}; - -#endif // NODEBASH_STATS || MEASURE_NODE_SIZE || COUNT_AST_OPERS - -#if COUNT_AST_OPERS -unsigned GenTree::s_gtNodeCounts[GT_COUNT + 1] = {0}; -#endif // COUNT_AST_OPERS - -/* static */ -void GenTree::InitNodeSize() -{ - /* Set all sizes to 'small' first */ - - for (unsigned op = 0; op <= GT_COUNT; op++) - { - GenTree::s_gtNodeSizes[op] = TREE_NODE_SZ_SMALL; - } - - // Now set all of the appropriate entries to 'large' - - // clang-format off - GenTree::s_gtNodeSizes[GT_CALL] = TREE_NODE_SZ_LARGE; -#if defined(FEATURE_SIMD) && defined(TARGET_XARCH) - GenTree::s_gtNodeSizes[GT_CNS_VEC] = TREE_NODE_SZ_LARGE; -#endif // FEATURE_SIMD && TARGET_XARCH - GenTree::s_gtNodeSizes[GT_CAST] = TREE_NODE_SZ_LARGE; - GenTree::s_gtNodeSizes[GT_FTN_ADDR] = TREE_NODE_SZ_LARGE; - GenTree::s_gtNodeSizes[GT_BOX] = TREE_NODE_SZ_LARGE; - GenTree::s_gtNodeSizes[GT_INDEX_ADDR] = TREE_NODE_SZ_LARGE; - GenTree::s_gtNodeSizes[GT_BOUNDS_CHECK] = TREE_NODE_SZ_SMALL; - GenTree::s_gtNodeSizes[GT_ARR_ELEM] = TREE_NODE_SZ_LARGE; - GenTree::s_gtNodeSizes[GT_RET_EXPR] = TREE_NODE_SZ_LARGE; - GenTree::s_gtNodeSizes[GT_FIELD_ADDR] = TREE_NODE_SZ_LARGE; - GenTree::s_gtNodeSizes[GT_CMPXCHG] = TREE_NODE_SZ_LARGE; - GenTree::s_gtNodeSizes[GT_QMARK] = TREE_NODE_SZ_LARGE; - GenTree::s_gtNodeSizes[GT_INTRINSIC] = TREE_NODE_SZ_LARGE; - GenTree::s_gtNodeSizes[GT_ALLOCOBJ] = TREE_NODE_SZ_LARGE; -#if USE_HELPERS_FOR_INT_DIV - GenTree::s_gtNodeSizes[GT_DIV] = TREE_NODE_SZ_LARGE; - GenTree::s_gtNodeSizes[GT_UDIV] = TREE_NODE_SZ_LARGE; - GenTree::s_gtNodeSizes[GT_MOD] = TREE_NODE_SZ_LARGE; - GenTree::s_gtNodeSizes[GT_UMOD] = TREE_NODE_SZ_LARGE; -#endif -#ifdef FEATURE_PUT_STRUCT_ARG_STK - // TODO-Throughput: This should not need to be a large node. The object info should be - // obtained from the child node. - GenTree::s_gtNodeSizes[GT_PUTARG_STK] = TREE_NODE_SZ_LARGE; -#if FEATURE_ARG_SPLIT - GenTree::s_gtNodeSizes[GT_PUTARG_SPLIT] = TREE_NODE_SZ_LARGE; -#endif // FEATURE_ARG_SPLIT -#endif // FEATURE_PUT_STRUCT_ARG_STK - - // This list of assertions should come to contain all GenTree subtypes that are declared - // "small". - assert(sizeof(GenTreeLclFld) <= GenTree::s_gtNodeSizes[GT_LCL_FLD]); - assert(sizeof(GenTreeLclVar) <= GenTree::s_gtNodeSizes[GT_LCL_VAR]); - - static_assert_no_msg(sizeof(GenTree) <= TREE_NODE_SZ_SMALL); - static_assert_no_msg(sizeof(GenTreeUnOp) <= TREE_NODE_SZ_SMALL); - static_assert_no_msg(sizeof(GenTreeOp) <= TREE_NODE_SZ_SMALL); - static_assert_no_msg(sizeof(GenTreeVal) <= TREE_NODE_SZ_SMALL); - static_assert_no_msg(sizeof(GenTreeIntConCommon) <= TREE_NODE_SZ_SMALL); - static_assert_no_msg(sizeof(GenTreePhysReg) <= TREE_NODE_SZ_SMALL); - static_assert_no_msg(sizeof(GenTreeIntCon) <= TREE_NODE_SZ_SMALL); - static_assert_no_msg(sizeof(GenTreeLngCon) <= TREE_NODE_SZ_SMALL); - static_assert_no_msg(sizeof(GenTreeDblCon) <= TREE_NODE_SZ_SMALL); - static_assert_no_msg(sizeof(GenTreeStrCon) <= TREE_NODE_SZ_SMALL); -#if defined(FEATURE_SIMD) -#ifdef TARGET_XARCH - static_assert_no_msg(sizeof(GenTreeVecCon) <= TREE_NODE_SZ_LARGE); // *** large node -#else - static_assert_no_msg(sizeof(GenTreeVecCon) <= TREE_NODE_SZ_SMALL); -#endif // TARGET_XARCH -#endif // FEATURE_SIMD - static_assert_no_msg(sizeof(GenTreeLclVarCommon) <= TREE_NODE_SZ_SMALL); - static_assert_no_msg(sizeof(GenTreeLclVar) <= TREE_NODE_SZ_SMALL); - static_assert_no_msg(sizeof(GenTreeLclFld) <= TREE_NODE_SZ_SMALL); - static_assert_no_msg(sizeof(GenTreeCC) <= TREE_NODE_SZ_SMALL); - static_assert_no_msg(sizeof(GenTreeOpCC) <= TREE_NODE_SZ_SMALL); -#ifdef TARGET_ARM64 - static_assert_no_msg(sizeof(GenTreeCCMP) <= TREE_NODE_SZ_SMALL); -#endif - static_assert_no_msg(sizeof(GenTreeConditional) <= TREE_NODE_SZ_SMALL); - static_assert_no_msg(sizeof(GenTreeCast) <= TREE_NODE_SZ_LARGE); // *** large node - static_assert_no_msg(sizeof(GenTreeBox) <= TREE_NODE_SZ_LARGE); // *** large node - static_assert_no_msg(sizeof(GenTreeFieldAddr) <= TREE_NODE_SZ_LARGE); // *** large node - static_assert_no_msg(sizeof(GenTreeFieldList) <= TREE_NODE_SZ_SMALL); - static_assert_no_msg(sizeof(GenTreeColon) <= TREE_NODE_SZ_SMALL); - static_assert_no_msg(sizeof(GenTreeCall) <= TREE_NODE_SZ_LARGE); // *** large node - static_assert_no_msg(sizeof(GenTreeCmpXchg) <= TREE_NODE_SZ_LARGE); // *** large node - static_assert_no_msg(sizeof(GenTreeFptrVal) <= TREE_NODE_SZ_LARGE); // *** large node - static_assert_no_msg(sizeof(GenTreeQmark) <= TREE_NODE_SZ_LARGE); // *** large node - static_assert_no_msg(sizeof(GenTreeIntrinsic) <= TREE_NODE_SZ_LARGE); // *** large node - static_assert_no_msg(sizeof(GenTreeIndexAddr) <= TREE_NODE_SZ_LARGE); // *** large node - static_assert_no_msg(sizeof(GenTreeArrLen) <= TREE_NODE_SZ_SMALL); - static_assert_no_msg(sizeof(GenTreeMDArr) <= TREE_NODE_SZ_SMALL); - static_assert_no_msg(sizeof(GenTreeBoundsChk) <= TREE_NODE_SZ_SMALL); - static_assert_no_msg(sizeof(GenTreeArrElem) <= TREE_NODE_SZ_LARGE); // *** large node - static_assert_no_msg(sizeof(GenTreeIndir) <= TREE_NODE_SZ_SMALL); - static_assert_no_msg(sizeof(GenTreeStoreInd) <= TREE_NODE_SZ_SMALL); - static_assert_no_msg(sizeof(GenTreeAddrMode) <= TREE_NODE_SZ_SMALL); - static_assert_no_msg(sizeof(GenTreeBlk) <= TREE_NODE_SZ_SMALL); - static_assert_no_msg(sizeof(GenTreeRetExpr) <= TREE_NODE_SZ_LARGE); // *** large node - static_assert_no_msg(sizeof(GenTreeILOffset) <= TREE_NODE_SZ_SMALL); - static_assert_no_msg(sizeof(GenTreePhiArg) <= TREE_NODE_SZ_SMALL); - static_assert_no_msg(sizeof(GenTreeAllocObj) <= TREE_NODE_SZ_LARGE); // *** large node -#ifndef FEATURE_PUT_STRUCT_ARG_STK - static_assert_no_msg(sizeof(GenTreePutArgStk) <= TREE_NODE_SZ_SMALL); -#else // FEATURE_PUT_STRUCT_ARG_STK - // TODO-Throughput: This should not need to be a large node. The object info should be - // obtained from the child node. - static_assert_no_msg(sizeof(GenTreePutArgStk) <= TREE_NODE_SZ_LARGE); -#if FEATURE_ARG_SPLIT - static_assert_no_msg(sizeof(GenTreePutArgSplit) <= TREE_NODE_SZ_LARGE); -#endif // FEATURE_ARG_SPLIT -#endif // FEATURE_PUT_STRUCT_ARG_STK - -#ifdef FEATURE_HW_INTRINSICS - static_assert_no_msg(sizeof(GenTreeHWIntrinsic) <= TREE_NODE_SZ_SMALL); -#endif // FEATURE_HW_INTRINSICS - // clang-format on -} - -size_t GenTree::GetNodeSize() const -{ - return GenTree::s_gtNodeSizes[gtOper]; -} - -#ifdef DEBUG -bool GenTree::IsNodeProperlySized() const -{ - size_t size; - - if (gtDebugFlags & GTF_DEBUG_NODE_SMALL) - { - size = TREE_NODE_SZ_SMALL; - } - else - { - assert(gtDebugFlags & GTF_DEBUG_NODE_LARGE); - size = TREE_NODE_SZ_LARGE; - } - - return GenTree::s_gtNodeSizes[gtOper] <= size; -} -#endif - -//------------------------------------------------------------------------ -// ReplaceWith: replace this with the src node. The source must be an isolated node -// and cannot be used after the replacement. -// -// Arguments: -// src - source tree, that replaces this. -// comp - the compiler instance to transfer annotations for arrays. -// -// Remarks: -// This should not be used for new code. -// -void GenTree::ReplaceWith(GenTree* src, Compiler* comp) -{ - // The source may be big only if the target is also a big node - assert((gtDebugFlags & GTF_DEBUG_NODE_LARGE) || GenTree::s_gtNodeSizes[src->gtOper] == TREE_NODE_SZ_SMALL); - - // The check is effective only if nodes have been already threaded. - assert((src->gtPrev == nullptr) && (src->gtNext == nullptr)); - - RecordOperBashing(OperGet(), src->OperGet()); // nop unless NODEBASH_STATS is enabled - - GenTree* prev = gtPrev; - GenTree* next = gtNext; - // The VTable pointer is copied intentionally here - memcpy((void*)this, (void*)src, src->GetNodeSize()); - this->gtPrev = prev; - this->gtNext = next; - -#ifdef DEBUG - gtSeqNum = 0; -#endif - DEBUG_DESTROY_NODE(src); -} - -/***************************************************************************** - * - * When 'NODEBASH_STATS' is enabled in "jit.h" we record all instances of - * an existing GenTree node having its operator changed. This can be useful - * for two (related) things - to see what is being bashed (and what isn't), - * and to verify that the existing choices for what nodes are marked 'large' - * are reasonable (to minimize "wasted" space). - * - * And yes, the hash function / logic is simplistic, but it is conflict-free - * and transparent for what we need. - */ - -#if NODEBASH_STATS - -#define BASH_HASH_SIZE 211 - -inline unsigned hashme(genTreeOps op1, genTreeOps op2) -{ - return ((op1 * 104729) ^ (op2 * 56569)) % BASH_HASH_SIZE; -} - -struct BashHashDsc -{ - uint32_t bhFullHash; // the hash value (unique for all old->new pairs) - uint32_t bhCount; // the same old->new bashings seen so far - uint8_t bhOperOld; // original gtOper - uint8_t bhOperNew; // new gtOper -}; - -static BashHashDsc BashHash[BASH_HASH_SIZE]; - -void GenTree::RecordOperBashing(genTreeOps operOld, genTreeOps operNew) -{ - unsigned hash = hashme(operOld, operNew); - BashHashDsc* desc = BashHash + hash; - - if (desc->bhFullHash != hash) - { - noway_assert(desc->bhCount == 0); // if this ever fires, need fix the hash fn - desc->bhFullHash = hash; - } - - desc->bhCount += 1; - desc->bhOperOld = operOld; - desc->bhOperNew = operNew; -} - -void GenTree::ReportOperBashing(FILE* f) -{ - unsigned total = 0; - - fflush(f); - - fprintf(f, "\n"); - fprintf(f, "Bashed gtOper stats:\n"); - fprintf(f, "\n"); - fprintf(f, " Old operator New operator #bytes old->new Count\n"); - fprintf(f, " ---------------------------------------------------------------\n"); - - for (unsigned h = 0; h < BASH_HASH_SIZE; h++) - { - unsigned count = BashHash[h].bhCount; - if (count == 0) - continue; - - unsigned opOld = BashHash[h].bhOperOld; - unsigned opNew = BashHash[h].bhOperNew; - - fprintf(f, " GT_%-13s -> GT_%-13s [size: %3u->%3u] %c %7u\n", OpName((genTreeOps)opOld), - OpName((genTreeOps)opNew), s_gtTrueSizes[opOld], s_gtTrueSizes[opNew], - (s_gtTrueSizes[opOld] < s_gtTrueSizes[opNew]) ? 'X' : ' ', count); - total += count; - } - fprintf(f, "\n"); - fprintf(f, "Total bashings: %u\n", total); - fprintf(f, "\n"); - - fflush(f); -} - -#endif // NODEBASH_STATS - -/*****************************************************************************/ - -#if MEASURE_NODE_SIZE - -void GenTree::DumpNodeSizes() -{ - // Dump the sizes of the various GenTree flavors - - FILE* fp = jitstdout(); - - fprintf(fp, "Small tree node size = %zu bytes\n", TREE_NODE_SZ_SMALL); - fprintf(fp, "Large tree node size = %zu bytes\n", TREE_NODE_SZ_LARGE); - fprintf(fp, "\n"); - - // Verify that node sizes are set kosherly and dump sizes - for (unsigned op = GT_NONE + 1; op < GT_COUNT; op++) - { - unsigned needSize = s_gtTrueSizes[op]; - unsigned nodeSize = s_gtNodeSizes[op]; - - const char* structNm = OpStructName((genTreeOps)op); - const char* operName = OpName((genTreeOps)op); - - bool repeated = false; - - // Have we seen this struct flavor before? - for (unsigned mop = GT_NONE + 1; mop < op; mop++) - { - if (strcmp(structNm, OpStructName((genTreeOps)mop)) == 0) - { - repeated = true; - break; - } - } - - // Don't repeat the same GenTree flavor unless we have an error - if (!repeated || needSize > nodeSize) - { - unsigned sizeChar = '?'; - - if (nodeSize == TREE_NODE_SZ_SMALL) - sizeChar = 'S'; - else if (nodeSize == TREE_NODE_SZ_LARGE) - sizeChar = 'L'; - - fprintf(fp, "GT_%-16s ... %-19s = %3u bytes (%c)", operName, structNm, needSize, sizeChar); - if (needSize > nodeSize) - { - fprintf(fp, " -- ERROR -- allocation is only %u bytes!", nodeSize); - } - else if (needSize <= TREE_NODE_SZ_SMALL && nodeSize == TREE_NODE_SZ_LARGE) - { - fprintf(fp, " ... could be small"); - } - - fprintf(fp, "\n"); - } - } -} - -#endif // MEASURE_NODE_SIZE - -//----------------------------------------------------------- -// begin: Get the iterator for the beginning of the locals list. -// -// Return Value: -// Iterator representing the beginning. -// -LocalsGenTreeList::iterator LocalsGenTreeList::begin() const -{ - GenTree* first = m_stmt->GetTreeList(); - assert((first == nullptr) || first->OperIsAnyLocal()); - return iterator(static_cast(first)); -} - -//----------------------------------------------------------- -// GetForwardEdge: Get the edge that points forward to a node. -// -// Arguments: -// node - The node the edge should be pointing at. -// -// Return Value: -// The edge, such that *edge == node. -// -GenTree** LocalsGenTreeList::GetForwardEdge(GenTreeLclVarCommon* node) -{ - if (node->gtPrev == nullptr) - { - assert(m_stmt->GetTreeList() == node); - return m_stmt->GetTreeListPointer(); - } - else - { - assert(node->gtPrev->gtNext == node); - return &node->gtPrev->gtNext; - } -} - -//----------------------------------------------------------- -// GetBackwardEdge: Get the edge that points backwards to a node. -// -// Arguments: -// node - The node the edge should be pointing at. -// -// Return Value: -// The edge, such that *edge == node. -// -GenTree** LocalsGenTreeList::GetBackwardEdge(GenTreeLclVarCommon* node) -{ - if (node->gtNext == nullptr) - { - assert(m_stmt->GetTreeListEnd() == node); - return m_stmt->GetTreeListEndPointer(); - } - else - { - assert(node->gtNext->gtPrev == node); - return &node->gtNext->gtPrev; - } -} - -//----------------------------------------------------------- -// Remove: Remove a specified node from the locals tree list. -// -// Arguments: -// node - the local node that should be part of this list. -// -void LocalsGenTreeList::Remove(GenTreeLclVarCommon* node) -{ - GenTree** forwardEdge = GetForwardEdge(node); - GenTree** backwardEdge = GetBackwardEdge(node); - - *forwardEdge = node->gtNext; - *backwardEdge = node->gtPrev; -} - -//----------------------------------------------------------- -// Replace: Replace a sequence of nodes with another (already linked) sequence of nodes. -// -// Arguments: -// firstNode - The first node, part of this locals tree list, to be replaced. -// lastNode - The last node, part of this locals tree list, to be replaced. -// newFirstNode - The start of the replacement sub list. -// newLastNode - The last node of the replacement sub list. -// -void LocalsGenTreeList::Replace(GenTreeLclVarCommon* firstNode, - GenTreeLclVarCommon* lastNode, - GenTreeLclVarCommon* newFirstNode, - GenTreeLclVarCommon* newLastNode) -{ - assert((newFirstNode != nullptr) && (newLastNode != nullptr)); - - GenTree** forwardEdge = GetForwardEdge(firstNode); - GenTree** backwardEdge = GetBackwardEdge(lastNode); - - GenTree* prev = firstNode->gtPrev; - GenTree* next = lastNode->gtNext; - - *forwardEdge = newFirstNode; - *backwardEdge = newLastNode; - newFirstNode->gtPrev = prev; - newLastNode->gtNext = next; -} - -//----------------------------------------------------------- -// TreeList: convenience method for enabling range-based `for` iteration over the -// execution order of the GenTree linked list, e.g.: -// for (GenTree* const tree : stmt->TreeList()) ... -// -// Only valid between fgSetBlockOrder and rationalization. See fgNodeThreading. -// -// Return Value: -// The tree list. -// -GenTreeList Statement::TreeList() const -{ - assert(JitTls::GetCompiler()->fgNodeThreading == NodeThreading::AllTrees); - return GenTreeList(GetTreeList()); -} - -//----------------------------------------------------------- -// LocalsTreeList: Manages the locals tree list and allows for range-based -// iteration. -// -// Only valid between local morph and forward sub. See fgNodeThreading. -// -// Return Value: -// The locals tree list. -// -LocalsGenTreeList Statement::LocalsTreeList() -{ - assert(JitTls::GetCompiler()->fgNodeThreading == NodeThreading::AllLocals); - return LocalsGenTreeList(this); -} - -/***************************************************************************** - * - * Walk all basic blocks and call the given function pointer for all tree - * nodes contained therein. - */ - -void Compiler::fgWalkAllTreesPre(fgWalkPreFn* visitor, void* pCallBackData) -{ - for (BasicBlock* const block : Blocks()) - { - for (Statement* const stmt : block->Statements()) - { - fgWalkTreePre(stmt->GetRootNodePointer(), visitor, pCallBackData); - } - } -} - -//----------------------------------------------------------- -// GetLayout: Get the struct layout for this node. -// -// Arguments: -// compiler - The Compiler instance -// -// Return Value: -// The struct layout of this node; it must have one. -// -// Notes: -// This is the "general" method for getting the layout, -// the more efficient node-specific ones should be used -// in case the node's oper is known. -// -ClassLayout* GenTree::GetLayout(Compiler* compiler) const -{ - assert(varTypeIsStruct(TypeGet())); - - CORINFO_CLASS_HANDLE structHnd = NO_CLASS_HANDLE; - switch (OperGet()) - { - case GT_LCL_VAR: - case GT_STORE_LCL_VAR: - return compiler->lvaGetDesc(AsLclVar())->GetLayout(); - - case GT_LCL_FLD: - case GT_STORE_LCL_FLD: - return AsLclFld()->GetLayout(); - - case GT_BLK: - case GT_STORE_BLK: - return AsBlk()->GetLayout(); - - case GT_COMMA: - return AsOp()->gtOp2->GetLayout(compiler); - -#ifdef FEATURE_HW_INTRINSICS - case GT_HWINTRINSIC: - return AsHWIntrinsic()->GetLayout(compiler); -#endif // FEATURE_HW_INTRINSICS - - case GT_CALL: - structHnd = AsCall()->gtRetClsHnd; - break; - - case GT_RET_EXPR: - structHnd = AsRetExpr()->gtInlineCandidate->gtRetClsHnd; - break; - - default: - unreached(); - } - - return compiler->typGetObjLayout(structHnd); -} - -//----------------------------------------------------------- -// CopyReg: Copy the _gtRegNum/gtRegTag fields. -// -// Arguments: -// from - GenTree node from which to copy -// -void GenTree::CopyReg(GenTree* from) -{ - _gtRegNum = from->_gtRegNum; - INDEBUG(gtRegTag = from->gtRegTag;) - - // Also copy multi-reg state if this is a call node - if (IsCall()) - { - assert(from->IsCall()); - this->AsCall()->CopyOtherRegs(from->AsCall()); - } - else if (IsCopyOrReload()) - { - this->AsCopyOrReload()->CopyOtherRegs(from->AsCopyOrReload()); - } -} - -//------------------------------------------------------------------ -// gtHasReg: Whether node been assigned a register by LSRA -// -// Arguments: -// comp - Compiler instance. Required for multi-reg lcl var; ignored otherwise. -// -// Return Value: -// Returns true if the node was assigned a register. -// -// In case of multi-reg call nodes, it is considered having a reg if regs are allocated for ALL its -// return values. -// REVIEW: why is this ALL and the other cases are ANY? Explain. -// -// In case of GT_COPY or GT_RELOAD of a multi-reg call, GT_COPY/GT_RELOAD is considered having a reg if it -// has a reg assigned to ANY of its positions. -// -// In case of multi-reg local vars, it is considered having a reg if it has a reg assigned for ANY -// of its positions. -// -bool GenTree::gtHasReg(Compiler* comp) const -{ - bool hasReg = false; - - if (IsMultiRegCall()) - { - const GenTreeCall* call = AsCall(); - const unsigned regCount = call->GetReturnTypeDesc()->GetReturnRegCount(); - - // A Multi-reg call node is said to have regs, if it has - // reg assigned to each of its result registers. - for (unsigned i = 0; i < regCount; ++i) - { - hasReg = (call->GetRegNumByIdx(i) != REG_NA); - if (!hasReg) - { - break; - } - } - } - else if (IsCopyOrReloadOfMultiRegCall()) - { - const GenTreeCopyOrReload* copyOrReload = AsCopyOrReload(); - const GenTreeCall* call = copyOrReload->gtGetOp1()->AsCall(); - const unsigned regCount = call->GetReturnTypeDesc()->GetReturnRegCount(); - - // A Multi-reg copy or reload node is said to have regs, - // if it has valid regs in any of the positions. - for (unsigned i = 0; i < regCount; ++i) - { - hasReg = (copyOrReload->GetRegNumByIdx(i) != REG_NA); - if (hasReg) - { - break; - } - } - } - else if (IsMultiRegLclVar()) - { - assert(comp != nullptr); - const GenTreeLclVar* lclNode = AsLclVar(); - const unsigned regCount = lclNode->GetFieldCount(comp); - // A Multi-reg local vars is said to have regs, - // if it has valid regs in any of the positions. - for (unsigned i = 0; i < regCount; i++) - { - hasReg = (lclNode->GetRegNumByIdx(i) != REG_NA); - if (hasReg) - { - break; - } - } - } - else - { - hasReg = (GetRegNum() != REG_NA); - } - - return hasReg; -} - -//----------------------------------------------------------------------------- -// GetRegisterDstCount: Get the number of registers defined by the node. -// -// Arguments: -// None -// -// Return Value: -// The number of registers that this node defines. -// -// Notes: -// This should not be called on a contained node. -// This does not look at the actual register assignments, if any, and so -// is valid after Lowering. -// -int GenTree::GetRegisterDstCount(Compiler* compiler) const -{ - assert(!isContained()); - if (!IsMultiRegNode()) - { - return IsValue() ? 1 : 0; - } - else if (IsMultiRegCall()) - { - return AsCall()->GetReturnTypeDesc()->GetReturnRegCount(); - } - else if (IsCopyOrReload()) - { - return gtGetOp1()->GetRegisterDstCount(compiler); - } -#if FEATURE_ARG_SPLIT - else if (OperIsPutArgSplit()) - { - return (const_cast(this))->AsPutArgSplit()->gtNumRegs; - } -#endif -#if !defined(TARGET_64BIT) - else if (OperIsMultiRegOp()) - { - assert(OperIs(GT_MUL_LONG)); - return 2; - } -#endif -#ifdef FEATURE_HW_INTRINSICS - else if (OperIsHWIntrinsic()) - { - assert(TypeIs(TYP_STRUCT)); - - const GenTreeHWIntrinsic* intrinsic = AsHWIntrinsic(); - const NamedIntrinsic intrinsicId = intrinsic->GetHWIntrinsicId(); - assert(HWIntrinsicInfo::IsMultiReg(intrinsicId)); - - return HWIntrinsicInfo::GetMultiRegCount(intrinsicId); - } -#endif // FEATURE_HW_INTRINSICS - - if (OperIsScalarLocal()) - { - return AsLclVar()->GetFieldCount(compiler); - } - assert(!"Unexpected multi-reg node"); - return 0; -} - -//----------------------------------------------------------------------------------- -// IsMultiRegNode: whether a node returning its value in more than one register -// -// Arguments: -// None -// -// Return Value: -// Returns true if this GenTree is a multi-reg node. -// -// Notes: -// All targets that support multi-reg ops of any kind also support multi-reg return -// values for calls. Should that change with a future target, this method will need -// to change accordingly. -// -bool GenTree::IsMultiRegNode() const -{ -#if FEATURE_MULTIREG_RET - if (IsMultiRegCall()) - { - return true; - } - -#if FEATURE_ARG_SPLIT - if (OperIsPutArgSplit()) - { - return true; - } -#endif - -#if !defined(TARGET_64BIT) - if (OperIsMultiRegOp()) - { - return AsMultiRegOp()->GetRegCount() > 1; - } -#endif -#endif // FEATURE_MULTIREG_RET - - if (OperIs(GT_COPY, GT_RELOAD)) - { - return true; - } - -#ifdef FEATURE_HW_INTRINSICS - if (OperIsHWIntrinsic()) - { - return HWIntrinsicInfo::IsMultiReg(AsHWIntrinsic()->GetHWIntrinsicId()); - } -#endif // FEATURE_HW_INTRINSICS - - if (IsMultiRegLclVar()) - { - return true; - } - return false; -} - -//----------------------------------------------------------------------------------- -// GetMultiRegCount: Return the register count for a multi-reg node. -// -// Arguments: -// comp - Compiler instance. Required for MultiRegLclVar, unused otherwise. -// -// Return Value: -// Returns the number of registers defined by this node. -// -unsigned GenTree::GetMultiRegCount(Compiler* comp) const -{ -#if FEATURE_MULTIREG_RET - if (IsMultiRegCall()) - { - return AsCall()->GetReturnTypeDesc()->GetReturnRegCount(); - } - -#if FEATURE_ARG_SPLIT - if (OperIsPutArgSplit()) - { - return AsPutArgSplit()->gtNumRegs; - } -#endif - -#if !defined(TARGET_64BIT) - if (OperIsMultiRegOp()) - { - return AsMultiRegOp()->GetRegCount(); - } -#endif -#endif // FEATURE_MULTIREG_RET - - if (OperIs(GT_COPY, GT_RELOAD)) - { - return AsCopyOrReload()->GetRegCount(); - } - -#ifdef FEATURE_HW_INTRINSICS - if (OperIsHWIntrinsic()) - { - return HWIntrinsicInfo::GetMultiRegCount(AsHWIntrinsic()->GetHWIntrinsicId()); - } -#endif // FEATURE_HW_INTRINSICS - - if (IsMultiRegLclVar()) - { - assert(comp != nullptr); - return AsLclVar()->GetFieldCount(comp); - } - - assert(!"GetMultiRegCount called with non-multireg node"); - return 1; -} - -#ifdef TARGET_ARM64 -//----------------------------------------------------------------------------------- -// NeedsConsecutiveRegisters: Checks if this tree node needs consecutive registers -// -// Return Value: -// Returns if the tree needs consecutive registers. -// -bool GenTree::NeedsConsecutiveRegisters() const -{ - return HWIntrinsicInfo::NeedsConsecutiveRegisters(AsHWIntrinsic()->GetHWIntrinsicId()); -} -#endif - -//--------------------------------------------------------------- -// gtGetContainedRegMask: Get the reg mask of the node including -// contained nodes (recursive). -// -// Arguments: -// None -// -// Return Value: -// Reg Mask of GenTree node. -// -regMaskTP GenTree::gtGetContainedRegMask() -{ - if (!isContained()) - { - return isUsedFromReg() ? gtGetRegMask() : RBM_NONE; - } - - regMaskTP mask = 0; - for (GenTree* operand : Operands()) - { - mask |= operand->gtGetContainedRegMask(); - } - return mask; -} - -//--------------------------------------------------------------- -// gtGetRegMask: Get the reg mask of the node. -// -// Arguments: -// None -// -// Return Value: -// Reg Mask of GenTree node. -// -regMaskTP GenTree::gtGetRegMask() const -{ - regMaskTP resultMask; - - if (IsMultiRegCall()) - { - resultMask = genRegMask(GetRegNum()); - resultMask |= AsCall()->GetOtherRegMask(); - } - else if (IsCopyOrReloadOfMultiRegCall()) - { - // A multi-reg copy or reload, will have valid regs for only those - // positions that need to be copied or reloaded. Hence we need - // to consider only those registers for computing reg mask. - - const GenTreeCopyOrReload* copyOrReload = AsCopyOrReload(); - const GenTreeCall* call = copyOrReload->gtGetOp1()->AsCall(); - const unsigned regCount = call->GetReturnTypeDesc()->GetReturnRegCount(); - - resultMask = RBM_NONE; - for (unsigned i = 0; i < regCount; ++i) - { - regNumber reg = copyOrReload->GetRegNumByIdx(i); - if (reg != REG_NA) - { - resultMask |= genRegMask(reg); - } - } - } -#if FEATURE_ARG_SPLIT - else if (compFeatureArgSplit() && OperIsPutArgSplit()) - { - const GenTreePutArgSplit* splitArg = AsPutArgSplit(); - const unsigned regCount = splitArg->gtNumRegs; - - resultMask = RBM_NONE; - for (unsigned i = 0; i < regCount; ++i) - { - regNumber reg = splitArg->GetRegNumByIdx(i); - assert(reg != REG_NA); - resultMask |= genRegMask(reg); - } - } -#endif // FEATURE_ARG_SPLIT - else - { - resultMask = genRegMask(GetRegNum()); - } - - return resultMask; -} - -void GenTreeFieldList::AddField(Compiler* compiler, GenTree* node, unsigned offset, var_types type) -{ - m_uses.AddUse(new (compiler, CMK_ASTNode) Use(node, offset, type)); - gtFlags |= node->gtFlags & GTF_ALL_EFFECT; -} - -void GenTreeFieldList::AddFieldLIR(Compiler* compiler, GenTree* node, unsigned offset, var_types type) -{ - m_uses.AddUse(new (compiler, CMK_ASTNode) Use(node, offset, type)); -} - -void GenTreeFieldList::InsertField(Compiler* compiler, Use* insertAfter, GenTree* node, unsigned offset, var_types type) -{ - m_uses.InsertUse(insertAfter, new (compiler, CMK_ASTNode) Use(node, offset, type)); - gtFlags |= node->gtFlags & GTF_ALL_EFFECT; -} - -void GenTreeFieldList::InsertFieldLIR( - Compiler* compiler, Use* insertAfter, GenTree* node, unsigned offset, var_types type) -{ - m_uses.InsertUse(insertAfter, new (compiler, CMK_ASTNode) Use(node, offset, type)); -} - -//--------------------------------------------------------------- -// IsHfaArg: Is this arg considered a homogeneous floating-point aggregate? -// -bool CallArgABIInformation::IsHfaArg() const -{ - if (GlobalJitOptions::compFeatureHfa) - { - return IsHfa(GetHfaElemKind()); - } - else - { - return false; - } -} - -//--------------------------------------------------------------- -// IsHfaRegArg: Is this an HFA argument passed in registers? -// -bool CallArgABIInformation::IsHfaRegArg() const -{ - if (GlobalJitOptions::compFeatureHfa) - { - return IsHfa(GetHfaElemKind()) && IsPassedInRegisters(); - } - else - { - return false; - } -} - -//--------------------------------------------------------------- -// GetHfaType: Get the type of each element of the HFA arg. -// -var_types CallArgABIInformation::GetHfaType() const -{ - if (GlobalJitOptions::compFeatureHfa) - { - return HfaTypeFromElemKind(GetHfaElemKind()); - } - else - { - return TYP_UNDEF; - } -} - -//--------------------------------------------------------------- -// SetHfaType: Set the type of each element of the HFA arg. -// -// Arguments: -// type - The new type for each element -// hfaSlots - How many registers are used by the HFA. -// -// Remarks: -// This can only be called after the passing mode of the argument (registers -// or stack) has been determined. When passing HFAs of doubles on ARM it is -// expected that `hfaSlots` refers to the number of float registers used, -// i.e. twice the number of doubles being passed. This function will convert -// that into double registers and set `NumRegs` appropriately. -// -void CallArgABIInformation::SetHfaType(var_types type, unsigned hfaSlots) -{ - if (GlobalJitOptions::compFeatureHfa) - { - if (type != TYP_UNDEF) - { - // We must already have set the passing mode. - assert(NumRegs != 0 || GetStackByteSize() != 0); - // We originally set numRegs according to the size of the struct, but if the size of the - // hfaType is not the same as the pointer size, we need to correct it. - // Note that hfaSlots is the number of registers we will use. For ARM, that is twice - // the number of "double registers". - unsigned numHfaRegs = hfaSlots; -#ifdef TARGET_ARM - if (type == TYP_DOUBLE) - { - // Must be an even number of registers. - assert((NumRegs & 1) == 0); - numHfaRegs = hfaSlots / 2; - } -#endif // TARGET_ARM - - if (!IsHfaArg()) - { - // We haven't previously set this; do so now. - CorInfoHFAElemType elemKind = HfaElemKindFromType(type); - SetHfaElemKind(elemKind); - // Ensure we've allocated enough bits. - assert(GetHfaElemKind() == elemKind); - if (IsPassedInRegisters()) - { - NumRegs = numHfaRegs; - } - } - else - { - // We've already set this; ensure that it's consistent. - if (IsPassedInRegisters()) - { - assert(NumRegs == numHfaRegs); - } - assert(type == HfaTypeFromElemKind(GetHfaElemKind())); - } - } - } -} - -//--------------------------------------------------------------- -// GetStackByteSize: Get the number of stack bytes used to pass this argument. -// -// Returns: -// For pure register arguments, this returns 0. -// For pure stack arguments, this returns ByteSize. -// For split arguments the return value is between 0 and ByteSize. -// -unsigned CallArgABIInformation::GetStackByteSize() const -{ - if (!IsSplit() && NumRegs > 0) - { - return 0; - } - - assert(!IsHfaArg() || !IsSplit()); - - assert(ByteSize > TARGET_POINTER_SIZE * NumRegs); - const unsigned stackByteSize = ByteSize - TARGET_POINTER_SIZE * NumRegs; - return stackByteSize; -} - -#ifdef DEBUG -void NewCallArg::ValidateTypes() -{ - assert(Compiler::impCheckImplicitArgumentCoercion(SignatureType, Node->TypeGet())); - - if (varTypeIsStruct(SignatureType)) - { - assert(SignatureClsHnd != NO_CLASS_HANDLE); - assert(SignatureType == Node->TypeGet()); - - if (SignatureType == TYP_STRUCT) - { - Compiler* comp = JitTls::GetCompiler(); - assert(ClassLayout::AreCompatible(comp->typGetObjLayout(SignatureClsHnd), Node->GetLayout(comp))); - } - } -} -#endif - -//--------------------------------------------------------------- -// IsArgAddedLate: Check if this is an argument that is added late, by -// `DetermineArgABIInformation`. -// -// Remarks: -// These arguments must be removed if ABI information needs to be -// reclassified by calling `DetermineArgABIInformation` as otherwise they -// will be readded. See `CallArgs::ResetFinalArgsAndABIInfo`. -// -// Note that the 'late' here is separate from CallArg::GetLateNode and -// friends. Late here refers to this being an argument that is added by morph -// instead of the importer. -// -bool CallArg::IsArgAddedLate() const -{ - // static_cast to "enum class" for old gcc support - switch (static_cast(m_wellKnownArg)) - { - case WellKnownArg::WrapperDelegateCell: - case WellKnownArg::VirtualStubCell: - case WellKnownArg::PInvokeCookie: - case WellKnownArg::PInvokeTarget: - case WellKnownArg::R2RIndirectionCell: - return true; - default: - return false; - } -} - -//--------------------------------------------------------------- -// IsUserArg: Check if this is an argument that can be treated as -// user-defined (in IL). -// -// Remarks: -// "this" and ShiftLow/ShiftHigh are recognized as user-defined -// -bool CallArg::IsUserArg() const -{ - switch (static_cast(m_wellKnownArg)) - { - case WellKnownArg::None: - case WellKnownArg::ShiftLow: - case WellKnownArg::ShiftHigh: - case WellKnownArg::ThisPointer: - return true; - default: - return false; - } -} - -#ifdef DEBUG -//--------------------------------------------------------------- -// CheckIsStruct: Verify that the struct ABI information is consistent with the IR node. -// -void CallArg::CheckIsStruct() -{ - GenTree* node = GetNode(); - if (varTypeIsStruct(GetSignatureType())) - { - if (!varTypeIsStruct(node) && !node->OperIs(GT_FIELD_LIST)) - { - // This is the case where we are passing a struct as a primitive type. - // On most targets, this is always a single register or slot. - // However, on ARM this could be two slots if it is TYP_DOUBLE. - bool isPassedAsPrimitiveType = - ((AbiInfo.NumRegs == 1) || ((AbiInfo.NumRegs == 0) && (AbiInfo.ByteSize <= TARGET_POINTER_SIZE))); -#ifdef TARGET_ARM - if (!isPassedAsPrimitiveType) - { - if (node->TypeGet() == TYP_DOUBLE && AbiInfo.NumRegs == 0 && (AbiInfo.GetStackSlotsNumber() == 2)) - { - isPassedAsPrimitiveType = true; - } - } -#endif // TARGET_ARM - assert(isPassedAsPrimitiveType); - } - } - else - { - assert(!varTypeIsStruct(node)); - } -} -#endif - -CallArgs::CallArgs() - : m_head(nullptr) - , m_lateHead(nullptr) - , m_argsStackSize(0) -#ifdef UNIX_X86_ABI - , m_stkSizeBytes(0) - , m_padStkAlign(0) -#endif - , m_hasThisPointer(false) - , m_hasRetBuffer(false) - , m_isVarArgs(false) - , m_abiInformationDetermined(false) - , m_newAbiInformationDetermined(false) - , m_hasRegArgs(false) - , m_hasStackArgs(false) - , m_argsComplete(false) - , m_needsTemps(false) -#ifdef UNIX_X86_ABI - , m_alignmentDone(false) -#endif -{ -} - -//--------------------------------------------------------------- -// FindByNode: Find the argument containing the specified early or late node. -// -// Parameters: -// node - The node to find. -// -// Returns: -// A pointer to the found CallArg, or otherwise nullptr. -// -CallArg* CallArgs::FindByNode(GenTree* node) -{ - assert(node != nullptr); - for (CallArg& arg : Args()) - { - if ((arg.GetEarlyNode() == node) || (arg.GetLateNode() == node)) - { - return &arg; - } - } - - return nullptr; -} - -//--------------------------------------------------------------- -// FindWellKnownArg: Find a specific well-known argument. -// -// Parameters: -// arg - The type of well-known argument. -// -// Returns: -// A pointer to the found CallArg, or null if it was not found. -// -// Remarks: -// For the 'this' arg or the return buffer arg there are more efficient -// alternatives available in `GetThisArg` and `GetRetBufferArg`. -// -CallArg* CallArgs::FindWellKnownArg(WellKnownArg arg) -{ - assert(arg != WellKnownArg::None); - for (CallArg& callArg : Args()) - { - if (callArg.GetWellKnownArg() == arg) - { - return &callArg; - } - } - - return nullptr; -} - -//--------------------------------------------------------------- -// GetThisArg: Get the this-pointer argument. -// -// Returns: -// A pointer to the 'this' arg, or nullptr if there is no such arg. -// -// Remarks: -// This is only the managed 'this' arg. We consider the 'this' pointer for -// unmanaged instance calling conventions as normal (non-this) arguments. -// -CallArg* CallArgs::GetThisArg() -{ - if (!HasThisPointer()) - { - return nullptr; - } - - // For calls that do have 'this' pointer the loop is cheap as this is - // almost always the first or second argument. - CallArg* result = FindWellKnownArg(WellKnownArg::ThisPointer); - assert(result && "Expected to find this pointer argument"); - return result; -} - -//--------------------------------------------------------------- -// GetRetBufferArg: Get the return buffer arg. -// -// Returns: -// A pointer to the ret-buffer arg, or nullptr if there is no such arg. -// -// Remarks: -// This is the actual (per-ABI) return buffer argument. On some ABIs this -// argument has special treatment. Notably on standard ARM64 calling -// convention it is passed in x8 (see `CallArgs::GetCustomRegister` for the -// exact conditions). -// -CallArg* CallArgs::GetRetBufferArg() -{ - if (!HasRetBuffer()) - { - return nullptr; - } - - CallArg* result = FindWellKnownArg(WellKnownArg::RetBuffer); - assert(result && "Expected to find ret buffer argument"); - return result; -} - -//--------------------------------------------------------------- -// GetArgByIndex: Get an argument with the specified index. -// -// Parameters: -// index - The index of the argument to find. -// -// Returns: -// A pointer to the argument. -// -// Remarks: -// This function assumes enough arguments exist. -// -CallArg* CallArgs::GetArgByIndex(unsigned index) -{ - CallArg* cur = m_head; - for (unsigned i = 0; i < index; i++) - { - assert((cur != nullptr) && "Not enough arguments in GetArgByIndex"); - cur = cur->GetNext(); - } - - return cur; -} - -//--------------------------------------------------------------- -// GetUserArgByIndex: Get an argument with the specified index. -// Unlike GetArgByIndex, this function ignores non-user args -// like r2r cells. -// -// Parameters: -// index - The index of the argument to find. -// -// Returns: -// A pointer to the argument. -// -// Remarks: -// This function assumes enough arguments exist. Also, see IsUserArg's -// comments -// -CallArg* CallArgs::GetUserArgByIndex(unsigned index) -{ - CallArg* cur = m_head; - assert((cur != nullptr) && "Not enough user arguments in GetUserArgByIndex"); - for (unsigned i = 0; i < index || !cur->IsUserArg();) - { - if (cur->IsUserArg()) - { - i++; - } - cur = cur->GetNext(); - assert((cur != nullptr) && "Not enough user arguments in GetUserArgByIndex"); - } - return cur; -} - -//--------------------------------------------------------------- -// GetIndex: Get the index for the specified argument. -// -// Parameters: -// arg - The argument to obtain the index of. -// -// Returns: -// The index. -// -unsigned CallArgs::GetIndex(CallArg* arg) -{ - unsigned i = 0; - for (CallArg& a : Args()) - { - if (&a == arg) - { - return i; - } - - i++; - } - - assert(!"Could not find argument in arg list"); - return (unsigned)-1; -} - -//--------------------------------------------------------------- -// Reverse: Reverse the specified subrange of arguments. -// -// Parameters: -// index - The index of the sublist to reverse. -// count - The length of the sublist to reverse. -// -// Remarks: -// This function is used for x86 stdcall/cdecl that passes arguments in the -// opposite order of the managed calling convention. -// -void CallArgs::Reverse(unsigned index, unsigned count) -{ - CallArg** headSlot = &m_head; - for (unsigned i = 0; i < index; i++) - { - assert(*headSlot != nullptr); - headSlot = &(*headSlot)->NextRef(); - } - - if (count > 1) - { - CallArg* newEnd = *headSlot; - CallArg* cur = (*headSlot)->GetNext(); - - for (unsigned i = 1; i < count; i++) - { - CallArg* next = cur->GetNext(); - cur->SetNext(*headSlot); - *headSlot = cur; - cur = next; - } - newEnd->SetNext(cur); - } -} - -//--------------------------------------------------------------- -// AddedWellKnownArg: Record details when a well known arg was added. -// -// Parameters: -// arg - The type of well-known arg that was just added. -// -// Remarks: -// This is used to improve performance of some common argument lookups. -// -void CallArgs::AddedWellKnownArg(WellKnownArg arg) -{ - switch (arg) - { - case WellKnownArg::ThisPointer: - m_hasThisPointer = true; - break; - case WellKnownArg::RetBuffer: - m_hasRetBuffer = true; - break; - default: - break; - } -} - -//--------------------------------------------------------------- -// RemovedWellKnownArg: Record details when a well known arg was removed. -// -// Parameters: -// arg - The type of well-known arg that was just removed. -// -void CallArgs::RemovedWellKnownArg(WellKnownArg arg) -{ - switch (arg) - { - case WellKnownArg::ThisPointer: - assert(FindWellKnownArg(arg) == nullptr); - m_hasThisPointer = false; - break; - case WellKnownArg::RetBuffer: - assert(FindWellKnownArg(arg) == nullptr); - m_hasRetBuffer = false; - break; - default: - break; - } -} - -//--------------------------------------------------------------- -// GetCustomRegister: Get the custom, non-standard register assignment for an argument. -// -// Parameters: -// comp - The compiler. -// cc - The calling convention. -// arg - The kind of argument. -// -// Returns: -// The custom register assignment, or REG_NA if this is a normally treated -// argument. -// -// Remarks: -// Many JIT helpers have custom calling conventions in order to improve -// performance. The pattern in those cases is to add a WellKnownArg for the -// arguments that are passed specially and teach this function how to pass -// them. Note that we only support passing such arguments in custom registers -// and generally never on stack. -// -regNumber CallArgs::GetCustomRegister(Compiler* comp, CorInfoCallConvExtension cc, WellKnownArg arg) -{ - switch (arg) - { -#if defined(TARGET_X86) || defined(TARGET_ARM) - // The x86 and arm32 CORINFO_HELP_INIT_PINVOKE_FRAME helpers have a custom calling convention. - case WellKnownArg::PInvokeFrame: - return REG_PINVOKE_FRAME; -#endif -#if defined(TARGET_ARM) - // A non-standard calling convention using wrapper delegate invoke is used - // on ARM, only, for wrapper delegates. It is used for VSD delegate calls - // where the VSD custom calling convention ABI requires passing R4, a - // callee-saved register, with a special value. Since R4 is a callee-saved - // register, its value needs to be preserved. Thus, the VM uses a wrapper - // delegate IL stub, which preserves R4 and also sets up R4 correctly for - // the VSD call. The VM is simply reusing an existing mechanism (wrapper - // delegate IL stub) to achieve its goal for delegate VSD call. See - // COMDelegate::NeedsWrapperDelegate() in the VM for details. - case WellKnownArg::WrapperDelegateCell: - return comp->virtualStubParamInfo->GetReg(); -#endif -#if defined(TARGET_X86) - // The x86 shift helpers have custom calling conventions and expect the lo - // part of the long to be in EAX and the hi part to be in EDX. - case WellKnownArg::ShiftLow: - return REG_LNGARG_LO; - case WellKnownArg::ShiftHigh: - return REG_LNGARG_HI; -#endif - case WellKnownArg::RetBuffer: - if (hasFixedRetBuffReg(cc)) - { - return theFixedRetBuffReg(cc); - } - - break; - - case WellKnownArg::VirtualStubCell: - return comp->virtualStubParamInfo->GetReg(); - - case WellKnownArg::PInvokeCookie: - return REG_PINVOKE_COOKIE_PARAM; - - case WellKnownArg::PInvokeTarget: - return REG_PINVOKE_TARGET_PARAM; - - case WellKnownArg::R2RIndirectionCell: - return REG_R2R_INDIRECT_PARAM; - - case WellKnownArg::ValidateIndirectCallTarget: - if (REG_VALIDATE_INDIRECT_CALL_ADDR != REG_ARG_0) - { - return REG_VALIDATE_INDIRECT_CALL_ADDR; - } - - break; - -#ifdef REG_DISPATCH_INDIRECT_CELL_ADDR - case WellKnownArg::DispatchIndirectCallTarget: - return REG_DISPATCH_INDIRECT_CALL_ADDR; -#endif - -#ifdef SWIFT_SUPPORT - case WellKnownArg::SwiftError: - assert(cc == CorInfoCallConvExtension::Swift); - return REG_SWIFT_ERROR; - - case WellKnownArg::SwiftSelf: - assert(cc == CorInfoCallConvExtension::Swift); - return REG_SWIFT_SELF; -#endif // SWIFT_SUPPORT - - default: - break; - } - - return REG_NA; -} - -//--------------------------------------------------------------- -// IsNonStandard: Check if an argument is passed with a non-standard calling -// convention. -// -// Parameters: -// comp - The compiler object. -// call - The call node containing these args. -// arg - The specific arg to check whether is non-standard. -// -// Returns: -// True if the argument is non-standard. -// -bool CallArgs::IsNonStandard(Compiler* comp, GenTreeCall* call, CallArg* arg) -{ - return GetCustomRegister(comp, call->GetUnmanagedCallConv(), arg->GetWellKnownArg()) != REG_NA; -} - -//--------------------------------------------------------------- -// PushFront: Create a new argument at the front of the argument list. -// -// Parameters: -// comp - The compiler. -// arg - The builder for the new arg. -// -// Returns: -// The created representative for the argument. -// -CallArg* CallArgs::PushFront(Compiler* comp, const NewCallArg& arg) -{ - CallArg* callArg = new (comp, CMK_CallArgs) CallArg(arg); - callArg->SetNext(m_head); - m_head = callArg; - AddedWellKnownArg(arg.WellKnownArg); - return callArg; -} - -//--------------------------------------------------------------- -// PushBack: Create a new argument at the back of the argument list. -// -// Parameters: -// comp - The compiler. -// arg - The builder for the new arg. -// -// Returns: -// The created representative for the argument. -// -CallArg* CallArgs::PushBack(Compiler* comp, const NewCallArg& arg) -{ - CallArg** slot = &m_head; - while (*slot != nullptr) - { - slot = &(*slot)->NextRef(); - } - - *slot = new (comp, CMK_CallArgs) CallArg(arg); - AddedWellKnownArg(arg.WellKnownArg); - return *slot; -} - -//--------------------------------------------------------------- -// InsertAfter: Create a new argument after another argument. -// -// Parameters: -// comp - The compiler. -// after - The existing argument to insert the new argument after. -// arg - The builder for the new arg. -// -// Returns: -// The created representative for the argument. -// -CallArg* CallArgs::InsertAfter(Compiler* comp, CallArg* after, const NewCallArg& arg) -{ -#ifdef DEBUG - bool found = false; - for (CallArg& arg : Args()) - { - if (&arg == after) - { - found = true; - break; - } - } - - assert(found && "Could not find arg to insert after in argument list"); -#endif - - return InsertAfterUnchecked(comp, after, arg); -} - -//--------------------------------------------------------------- -// InsertAfterUnchecked: Create a new argument after another argument, without debug checks. -// -// Parameters: -// comp - The compiler. -// after - The existing argument to insert the new argument after. -// arg - The builder for the new arg. -// -// Returns: -// The created representative for the argument. -// -CallArg* CallArgs::InsertAfterUnchecked(Compiler* comp, CallArg* after, const NewCallArg& arg) -{ - CallArg* newArg = new (comp, CMK_CallArgs) CallArg(arg); - newArg->SetNext(after->GetNext()); - after->SetNext(newArg); - AddedWellKnownArg(arg.WellKnownArg); - return newArg; -} - -//--------------------------------------------------------------- -// InsertInstParam: Insert an instantiation parameter/generic context argument. -// -// Parameters: -// comp - The compiler. -// node - The IR node for the instantiation parameter. -// -// Returns: -// The created representative for the argument. -// -// Remarks: -// The instantiation parameter is a normal parameter, but its position in the -// arg list depends on a few factors. It is inserted at the end on x86 and on -// other platforms must always come after the ret-buffer and the 'this' -// argument. -// -CallArg* CallArgs::InsertInstParam(Compiler* comp, GenTree* node) -{ - NewCallArg newArg = NewCallArg::Primitive(node).WellKnown(WellKnownArg::InstParam); - - if (Target::g_tgtArgOrder == Target::ARG_ORDER_R2L) - { - CallArg* retBufferArg = GetRetBufferArg(); - if (retBufferArg != nullptr) - { - return InsertAfter(comp, retBufferArg, newArg); - } - else - { - return InsertAfterThisOrFirst(comp, newArg); - } - } - else - { - return PushBack(comp, newArg); - } -} - -//--------------------------------------------------------------- -// InsertAfterThisOrFirst: Insert an argument after 'this' if the call has a -// 'this' argument, or otherwise first. -// -// Parameters: -// comp - The compiler. -// arg - The builder for the new arg. -// -// Returns: -// The created representative for the argument. -// -CallArg* CallArgs::InsertAfterThisOrFirst(Compiler* comp, const NewCallArg& arg) -{ - CallArg* thisArg = GetThisArg(); - if (thisArg != nullptr) - { - return InsertAfter(comp, thisArg, arg); - } - else - { - return PushFront(comp, arg); - } -} - -//--------------------------------------------------------------- -// PushLateBack: Insert an argument at the end of the 'late' argument list. -// -// Parameters: -// arg - The arg to add to the late argument list. -// -// Remarks: -// This function should only be used if adding arguments after the call has -// already been morphed. -// -void CallArgs::PushLateBack(CallArg* arg) -{ - CallArg** slot = &m_lateHead; - while (*slot != nullptr) - { - slot = &(*slot)->LateNextRef(); - } - - *slot = arg; -} - -//--------------------------------------------------------------- -// Remove: Remove an argument from the argument list. -// -// Parameters: -// arg - The arg to remove. -// -// Remarks: -// This function cannot be used after morph. It will also invalidate ABI -// information, so it is expected that `CallArgs::AddFinalArgsAndDetermineABIInfo` -// was not called yet or that `CallArgs::ResetFinalArgsAndABIInfo` has been -// called prior to this. -// -void CallArgs::Remove(CallArg* arg) -{ - assert(!m_abiInformationDetermined && !m_argsComplete); - - CallArg** slot = &m_head; - while (*slot != nullptr) - { - if (*slot == arg) - { - *slot = arg->GetNext(); - RemovedWellKnownArg(arg->GetWellKnownArg()); - return; - } - - slot = &(*slot)->NextRef(); - } - - assert(!"Did not find arg to remove in CallArgs::Remove"); -} - -#ifdef TARGET_XARCH -//--------------------------------------------------------------- -// NeedsVzeroupper: Determines if the call needs a vzeroupper emitted before it is invoked -// -// Parameters: -// comp - the compiler -// -// Returns: -// true if a vzeroupper needs to be emitted; otherwise, false -// -bool GenTreeCall::NeedsVzeroupper(Compiler* comp) -{ - if (!comp->canUseVexEncoding()) - { - return false; - } - - bool needsVzeroupper = false; - - if (IsPInvoke()) - { - // The Intel optimization manual guidance in `3.11.5.3 Fixing Instruction Slowdowns` states: - // Insert a VZEROUPPER to tell the hardware that the state of the higher registers is clean - // between the VEX and the legacy SSE instructions. Often the best way to do this is to insert a - // VZEROUPPER before returning from any function that uses VEX (that does not produce a VEX - // register) and before any call to an unknown function. - - switch (gtCallType) - { - case CT_USER_FUNC: - case CT_INDIRECT: - { - // Since P/Invokes are not compiled by the runtime, they are typically "unknown" since they - // may use the legacy encoding. This includes both CT_USER_FUNC and CT_INDIRECT - - needsVzeroupper = true; - break; - } - - case CT_HELPER: - { - // Most helpers are well known to not use any floating-point or SIMD logic internally, but - // a few do exist so we need to ensure they are handled. They are identified by taking or - // returning a floating-point or SIMD type, regardless of how it is actually passed/returned. - - if (varTypeUsesFloatReg(this)) - { - needsVzeroupper = true; - break; - } - else - { - for (CallArg& arg : gtArgs.Args()) - { - if (varTypeUsesFloatReg(arg.GetSignatureType())) - { - needsVzeroupper = true; - break; - } - } - } - break; - } - - default: - { - unreached(); - } - } - } - - // Other special cases - // - if (!needsVzeroupper && IsHelperCall(comp, CORINFO_HELP_BULK_WRITEBARRIER)) - { - needsVzeroupper = true; - } - - return needsVzeroupper; -} -#endif // TARGET_XARCH - -//--------------------------------------------------------------- -// GetOtherRegMask: Get the reg mask of gtOtherRegs of call node -// -// Arguments: -// None -// -// Return Value: -// Reg mask of gtOtherRegs of call node. -// -regMaskTP GenTreeCall::GetOtherRegMask() const -{ - regMaskTP resultMask = RBM_NONE; - -#if FEATURE_MULTIREG_RET - for (unsigned i = 0; i < MAX_RET_REG_COUNT - 1; ++i) - { - if (gtOtherRegs[i] != REG_NA) - { - resultMask |= genRegMask((regNumber)gtOtherRegs[i]); - continue; - } - break; - } -#endif - - return resultMask; -} - -//------------------------------------------------------------------------- -// IsPure: -// Returns true if this call is pure. For now, this uses the same -// definition of "pure" that is that used by HelperCallProperties: a -// pure call does not read or write any aliased (e.g. heap) memory or -// have other global side effects (e.g. class constructors, finalizers), -// but is allowed to throw an exception. -// -// NOTE: this call currently only returns true if the call target is a -// helper method that is known to be pure. No other analysis is -// performed. -// -// Arguments: -// compiler - the compiler context. -// -// Returns: -// True if the call is pure; false otherwise. -// -bool GenTreeCall::IsPure(Compiler* compiler) const -{ - if (IsHelperCall()) - { - return compiler->s_helperCallProperties.IsPure(compiler->eeGetHelperNum(gtCallMethHnd)); - } - // If needed, we can annotate other special intrinsic methods as pure as well. - return IsSpecialIntrinsic(compiler, NI_System_Type_GetTypeFromHandle); -} - -//------------------------------------------------------------------------------ -// getArrayLengthFromAllocation: Return the array length for an array allocation -// helper call. -// -// Arguments: -// tree - The array allocation helper call. -// block - tree's basic block. -// -// Return Value: -// Return the array length node. - -GenTree* Compiler::getArrayLengthFromAllocation(GenTree* tree DEBUGARG(BasicBlock* block)) -{ - assert(tree != nullptr); - - GenTree* arrayLength = nullptr; - - if (tree->OperGet() == GT_CALL) - { - GenTreeCall* call = tree->AsCall(); - - if (call->IsHelperCall()) - { - CorInfoHelpFunc helper = eeGetHelperNum(call->gtCallMethHnd); - switch (helper) - { - case CORINFO_HELP_NEWARR_1_MAYBEFROZEN: - case CORINFO_HELP_NEWARR_1_DIRECT: - case CORINFO_HELP_NEWARR_1_OBJ: - case CORINFO_HELP_NEWARR_1_VC: - case CORINFO_HELP_NEWARR_1_ALIGN8: - { - // This is an array allocation site. Grab the array length node. - arrayLength = call->gtArgs.GetUserArgByIndex(1)->GetNode(); - break; - } - - default: - break; - } - - assert((arrayLength == nullptr) || ((optMethodFlags & OMF_HAS_NEWARRAY) != 0)); - } - } - - if (arrayLength != nullptr) - { - arrayLength = arrayLength->OperIsPutArg() ? arrayLength->gtGetOp1() : arrayLength; - } - - return arrayLength; -} - -//------------------------------------------------------------------------- -// SetSingleInlineCandidateInfo: set a single inline candidate info in the current call. -// -// Arguments: -// candidateInfo - inline candidate info -// -void GenTreeCall::SetSingleInlineCandidateInfo(InlineCandidateInfo* candidateInfo) -{ - if (candidateInfo != nullptr) - { - gtFlags |= GTF_CALL_INLINE_CANDIDATE; - gtInlineInfoCount = 1; - } - else - { - gtInlineInfoCount = 0; - gtFlags &= ~GTF_CALL_INLINE_CANDIDATE; - } - gtInlineCandidateInfo = candidateInfo; - ClearGuardedDevirtualizationCandidate(); -} - -//------------------------------------------------------------------------- -// GetGDVCandidateInfo: Get GDV candidate info in the current call by index. -// -// Return Value: -// GDV candidate info -// -InlineCandidateInfo* GenTreeCall::GetGDVCandidateInfo(uint8_t index) -{ - assert(index < gtInlineInfoCount); - if (gtInlineInfoCount > 1) - { - // In this case we should access it through gtInlineCandidateInfoList - return gtInlineCandidateInfoList->at(index); - } - return gtInlineCandidateInfo; -} - -//------------------------------------------------------------------------- -// AddGDVCandidateInfo: Record a guarded devirtualization (GDV) candidate info -// for this call. A call can't have more than MAX_GDV_TYPE_CHECKS number of candidates -// -// Arguments: -// comp - Compiler instance -// candidateInfo - GDV candidate info -// -void GenTreeCall::AddGDVCandidateInfo(Compiler* comp, InlineCandidateInfo* candidateInfo) -{ - assert((gtCallMoreFlags & GTF_CALL_M_GUARDED_DEVIRT_EXACT) == 0); - assert(gtInlineInfoCount < MAX_GDV_TYPE_CHECKS); - assert(candidateInfo != nullptr); - - if (gtInlineInfoCount == 0) - { - // Most calls are monomorphic, so we don't need to allocate a vector - gtInlineCandidateInfo = candidateInfo; - } - else if (gtInlineInfoCount == 1) - { - // Upgrade gtInlineCandidateInfo to gtInlineCandidateInfoList (vector) - CompAllocator allocator = comp->getAllocator(CMK_Inlining); - InlineCandidateInfo* firstCandidate = gtInlineCandidateInfo; - gtInlineCandidateInfoList = new (allocator) jitstd::vector(allocator); - gtInlineCandidateInfoList->push_back(firstCandidate); - gtInlineCandidateInfoList->push_back(candidateInfo); - } - else - { - gtInlineCandidateInfoList->push_back(candidateInfo); - } - gtCallMoreFlags |= GTF_CALL_M_GUARDED_DEVIRT; - gtInlineInfoCount++; -} - -//------------------------------------------------------------------------- -// RemoveGDVCandidateInfo: Remove a guarded devirtualization (GDV) candidate info -// by its index. Index must not be greater than gtInlineInfoCount -// the call will be marked as "has no inline candidates" if the last candidate is removed -// -// Arguments: -// comp - Compiler instance -// index - GDV candidate to remove -// -void GenTreeCall::RemoveGDVCandidateInfo(Compiler* comp, uint8_t index) -{ - // We change the number of candidates so it's no longer "doesn't need a fallback" - gtCallMoreFlags &= ~GTF_CALL_M_GUARDED_DEVIRT_EXACT; - - assert(index < gtInlineInfoCount); - - if (gtInlineInfoCount == 1) - { - // No longer have any inline candidates - ClearInlineInfo(); - assert(gtInlineInfoCount == 0); - return; - } - gtInlineCandidateInfoList->erase(gtInlineCandidateInfoList->begin() + index); - gtInlineInfoCount--; - - // Downgrade gtInlineCandidateInfoList to gtInlineCandidateInfo - if (gtInlineInfoCount == 1) - { - gtInlineCandidateInfo = gtInlineCandidateInfoList->at(0); - } -} - -//------------------------------------------------------------------------- -// HasSideEffects: -// Returns true if this call has any side effects. All non-helpers are considered to have side-effects. Only helpers -// that do not mutate the heap, do not run constructors, may not throw, and are either a) pure or b) non-finalizing -// allocation functions are considered side-effect-free. -// -// Arguments: -// compiler - the compiler instance -// ignoreExceptions - when `true`, ignores exception side effects -// ignoreCctors - when `true`, ignores class constructor side effects -// -// Return Value: -// true if this call has any side-effects; false otherwise. -bool GenTreeCall::HasSideEffects(Compiler* compiler, bool ignoreExceptions, bool ignoreCctors) const -{ - // Generally all GT_CALL nodes are considered to have side-effects, but we may have extra information about helper - // calls that can prove them side-effect-free. - if (!IsHelperCall()) - { - // If needed, we can annotate other special intrinsic methods as side effect free as well. - if (IsSpecialIntrinsic(compiler, NI_System_Type_GetTypeFromHandle)) - { - return false; - } - - return true; - } - - CorInfoHelpFunc helper = compiler->eeGetHelperNum(gtCallMethHnd); - HelperCallProperties& helperProperties = compiler->s_helperCallProperties; - - // We definitely care about the side effects if MutatesHeap is true - if (helperProperties.MutatesHeap(helper)) - { - return true; - } - - // Unless we have been instructed to ignore cctors (CSE, for example, ignores cctors), consider them side effects. - if (!ignoreCctors && helperProperties.MayRunCctor(helper)) - { - return true; - } - - // Consider array allocators side-effect free for constant length (if it's not negative and fits into i32) - if (helperProperties.IsAllocator(helper)) - { - GenTree* arrLen = compiler->getArrayLengthFromAllocation((GenTree*)this DEBUGARG(nullptr)); - // if arrLen is nullptr it means it wasn't an array allocator - if ((arrLen != nullptr) && arrLen->IsIntCnsFitsInI32()) - { - ssize_t cns = arrLen->AsIntConCommon()->IconValue(); - if ((cns >= 0) && (cns <= CORINFO_Array_MaxLength)) - { - return false; - } - } - } - - // If we also care about exceptions then check if the helper can throw - if (!ignoreExceptions && !helperProperties.NoThrow(helper)) - { - return true; - } - - // If this is not a Pure helper call or an allocator (that will not need to run a finalizer) - // then this call has side effects. - return !helperProperties.IsPure(helper) && - (!helperProperties.IsAllocator(helper) || ((gtCallMoreFlags & GTF_CALL_M_ALLOC_SIDE_EFFECTS) != 0)); -} - -//------------------------------------------------------------------------- -// HasNonStandardAddedArgs: Return true if the method has non-standard args added to the call -// argument list during argument morphing (fgMorphArgs), e.g., passed in R10 or R11 on AMD64. -// See also GetNonStandardAddedArgCount(). -// -// Arguments: -// compiler - the compiler instance -// -// Return Value: -// true if there are any such args, false otherwise. -// -bool GenTreeCall::HasNonStandardAddedArgs(Compiler* compiler) const -{ - return GetNonStandardAddedArgCount(compiler) != 0; -} - -//------------------------------------------------------------------------- -// GetNonStandardAddedArgCount: Get the count of non-standard arguments that have been added -// during call argument morphing (fgMorphArgs). Do not count non-standard args that are already -// counted in the argument list prior to morphing. -// -// This function is used to help map the caller and callee arguments during tail call setup. -// -// Arguments: -// compiler - the compiler instance -// -// Return Value: -// The count of args, as described. -// -// Notes: -// It would be more general to have fgMorphArgs set a bit on the call node when such -// args are added to a call, and a bit on each such arg, and then have this code loop -// over the call args when the special call bit is set, counting the args with the special -// arg bit. This seems pretty heavyweight, though. Instead, this logic needs to be kept -// in sync with fgMorphArgs. -// -int GenTreeCall::GetNonStandardAddedArgCount(Compiler* compiler) const -{ - if (IsUnmanaged() && !compiler->opts.ShouldUsePInvokeHelpers()) - { - // R11 = PInvoke cookie param - return 1; - } - else if (IsVirtualStub()) - { - // R11 = Virtual stub param - return 1; - } - else if ((gtCallType == CT_INDIRECT) && (gtCallCookie != nullptr)) - { - // R10 = PInvoke target param - // R11 = PInvoke cookie param - return 2; - } - return 0; -} - -//------------------------------------------------------------------------- -// IsHelperCall: Determine if this GT_CALL node is a specific helper call. -// -// Arguments: -// compiler - the compiler instance so that we can call eeFindHelper -// -// Return Value: -// Returns true if this GT_CALL node is a call to the specified helper. -// -bool GenTreeCall::IsHelperCall(Compiler* compiler, unsigned helper) const -{ - return IsHelperCall(compiler->eeFindHelper(helper)); -} - -//------------------------------------------------------------------------- -// IsRuntimeLookupHelperCall: Determine if this GT_CALL node represents a runtime lookup helper call. -// -// Arguments: -// compiler - the compiler instance so that we can call eeGetHelperNum -// -// Return Value: -// Returns true if this GT_CALL node represents a runtime lookup helper call. -// -bool GenTreeCall::IsRuntimeLookupHelperCall(Compiler* compiler) const -{ - if (!IsHelperCall()) - { - return false; - } - - switch (compiler->eeGetHelperNum(gtCallMethHnd)) - { - case CORINFO_HELP_RUNTIMEHANDLE_METHOD: - case CORINFO_HELP_RUNTIMEHANDLE_CLASS: - return true; - default: - return false; - } -} - -//------------------------------------------------------------------------- -// IsSpecialIntrinsic: Determine if this GT_CALL node is a specific intrinsic. -// -// Arguments: -// compiler - the compiler instance so that we can call lookupNamedIntrinsic -// ni - the intrinsic id -// -// Return Value: -// Returns true if this GT_CALL node is a special intrinsic call. -// -bool GenTreeCall::IsSpecialIntrinsic(Compiler* compiler, NamedIntrinsic ni) const -{ - return IsSpecialIntrinsic() && (compiler->lookupNamedIntrinsic(gtCallMethHnd) == ni); -} - -//------------------------------------------------------------------------- -// GetHelperNum: Get the helper identifier for this call. -// -// Return Value: -// CORINFO_HELP_UNDEF if this call is not a helper call, appropriate -// CorInfoHelpFunc otherwise. -// -CorInfoHelpFunc GenTreeCall::GetHelperNum() const -{ - return IsHelperCall() ? Compiler::eeGetHelperNum(gtCallMethHnd) : CORINFO_HELP_UNDEF; -} - -//-------------------------------------------------------------------------- -// Equals: Check if 2 CALL nodes are equal. -// -// Arguments: -// c1 - The first call node -// c2 - The second call node -// -// Return Value: -// true if the 2 CALL nodes have the same type and operands -// -bool GenTreeCall::Equals(GenTreeCall* c1, GenTreeCall* c2) -{ - assert(c1->OperGet() == c2->OperGet()); - - if (c1->TypeGet() != c2->TypeGet()) - { - return false; - } - - if (c1->gtCallType != c2->gtCallType) - { - return false; - } - - if (c1->gtCallType != CT_INDIRECT) - { - if (c1->gtCallMethHnd != c2->gtCallMethHnd) - { - return false; - } - - if (c1->IsHelperCall() && ((c1->gtCallMoreFlags & GTF_CALL_M_CAST_OBJ_NONNULL) != - (c2->gtCallMoreFlags & GTF_CALL_M_CAST_OBJ_NONNULL))) - { - return false; - } - -#ifdef FEATURE_READYTORUN - if (c1->gtEntryPoint.addr != c2->gtEntryPoint.addr) - { - return false; - } -#endif - - if ((c1->gtCallType == CT_USER_FUNC) && - ((c1->gtFlags & GTF_CALL_VIRT_KIND_MASK) != (c2->gtFlags & GTF_CALL_VIRT_KIND_MASK))) - { - return false; - } - } - else - { - if (!Compare(c1->gtCallAddr, c2->gtCallAddr)) - { - return false; - } - } - - { - CallArgs::ArgIterator i1 = c1->gtArgs.Args().begin(); - CallArgs::ArgIterator end1 = c1->gtArgs.Args().end(); - CallArgs::ArgIterator i2 = c2->gtArgs.Args().begin(); - CallArgs::ArgIterator end2 = c2->gtArgs.Args().end(); - - for (; (i1 != end1) && (i2 != end2); ++i1, ++i2) - { - if (!Compare(i1->GetEarlyNode(), i2->GetEarlyNode())) - { - return false; - } - - if (!Compare(i1->GetLateNode(), i2->GetLateNode())) - { - return false; - } - } - - if ((i1 != end1) || (i2 != end2)) - { - return false; - } - } - - if (!Compare(c1->gtControlExpr, c2->gtControlExpr)) - { - return false; - } - - return true; -} - -//-------------------------------------------------------------------------- -// ResetFinalArgsAndABIInfo: Reset ABI information classified for arguments, -// removing late-added arguments. -// -// Remarks: -// This function can be called between `CallArgs::AddFinalArgsAndDetermineABIInfo` -// and actually finishing the morphing of arguments. It cannot be called once -// the arguments have finished morphing. -// -void CallArgs::ResetFinalArgsAndABIInfo() -{ - m_newAbiInformationDetermined = false; - - if (!IsAbiInformationDetermined()) - { - return; - } - - // `CallArgs::AddFinalArgsAndDetermineABIInfo` not only sets up arg info, it - // also adds non-standard args to the IR, and we need to remove that extra - // IR so it doesn't get added again. - CallArg** link = &m_head; - - // We cannot handle this being called after fgMorphArgs, only between - // CallArgs::AddFinalArgsAndDetermineABIInfo and finishing args. - assert(!m_argsComplete); - - while ((*link) != nullptr) - { - // Check if this is an argument added by AddFinalArgsAndDetermineABIInfo. - if ((*link)->IsArgAddedLate()) - { - JITDUMP("Removing arg %s [%06u] to prepare for re-morphing call\n", - getWellKnownArgName((*link)->GetWellKnownArg()), Compiler::dspTreeID((*link)->GetNode())); - - *link = (*link)->GetNext(); - } - else - { - link = &(*link)->NextRef(); - } - } - - m_abiInformationDetermined = false; -} - -#if !defined(FEATURE_PUT_STRUCT_ARG_STK) -unsigned GenTreePutArgStk::GetStackByteSize() const -{ - return genTypeSize(genActualType(gtOp1->gtType)); -} -#endif // !defined(FEATURE_PUT_STRUCT_ARG_STK) - -/***************************************************************************** - * - * Returns non-zero if the two trees are identical. - */ - -bool GenTree::Compare(GenTree* op1, GenTree* op2, bool swapOK) -{ - genTreeOps oper; - unsigned kind; - -AGAIN: - - if (op1 == nullptr) - { - return (op2 == nullptr); - } - if (op2 == nullptr) - { - return false; - } - if (op1 == op2) - { - return true; - } - - oper = op1->OperGet(); - - /* The operators must be equal */ - - if (oper != op2->gtOper) - { - return false; - } - - /* The types must be equal */ - - if (op1->gtType != op2->gtType) - { - return false; - } - - /* Overflow must be equal */ - if (op1->gtOverflowEx() != op2->gtOverflowEx()) - { - return false; - } - - /* Sensible flags must be equal */ - if ((op1->gtFlags & (GTF_UNSIGNED)) != (op2->gtFlags & (GTF_UNSIGNED))) - { - return false; - } - if (op1->OperIs(GT_MOD, GT_UMOD, GT_DIV, GT_UDIV)) - { - if ((op1->gtFlags & (GTF_DIV_MOD_NO_BY_ZERO | GTF_DIV_MOD_NO_OVERFLOW)) != - (op2->gtFlags & (GTF_DIV_MOD_NO_BY_ZERO | GTF_DIV_MOD_NO_OVERFLOW))) - { - return false; - } - } - - /* Figure out what kind of nodes we're comparing */ - - kind = op1->OperKind(); - - /* Is this a constant node? */ - - if (op1->OperIsConst()) - { - switch (oper) - { - case GT_CNS_INT: - if (op1->AsIntCon()->gtIconVal == op2->AsIntCon()->gtIconVal) - { - return true; - } - break; - - case GT_CNS_LNG: - { - if (op1->AsLngCon()->gtLconVal == op2->AsLngCon()->gtLconVal) - { - return true; - } - break; - } - - case GT_CNS_DBL: - { - if (op1->AsDblCon()->isBitwiseEqual(op2->AsDblCon())) - { - return true; - } - break; - } - - case GT_CNS_STR: - if ((op1->AsStrCon()->gtSconCPX == op2->AsStrCon()->gtSconCPX) && - (op1->AsStrCon()->gtScpHnd == op2->AsStrCon()->gtScpHnd)) - { - return true; - } - break; - -#if defined(FEATURE_SIMD) - case GT_CNS_VEC: - { - if (GenTreeVecCon::Equals(op1->AsVecCon(), op2->AsVecCon())) - { - return true; - } - break; - } -#endif // FEATURE_SIMD - -#if defined(FEATURE_MASKED_HW_INTRINSICS) - case GT_CNS_MSK: - { - if (GenTreeMskCon::Equals(op1->AsMskCon(), op2->AsMskCon())) - { - return true; - } - break; - } -#endif // FEATURE_MASKED_HW_INTRINSICS - - default: - break; - } - - return false; - } - - /* Is this a leaf node? */ - - if (kind & GTK_LEAF) - { - switch (oper) - { - case GT_LCL_FLD: - if (op1->AsLclFld()->GetLayout() != op2->AsLclFld()->GetLayout()) - { - break; - } - FALLTHROUGH; - - case GT_LCL_ADDR: - if (op1->AsLclFld()->GetLclOffs() != op2->AsLclFld()->GetLclOffs()) - { - break; - } - FALLTHROUGH; - - case GT_LCL_VAR: - if (op1->AsLclVarCommon()->GetLclNum() != op2->AsLclVarCommon()->GetLclNum()) - { - break; - } - return true; - - case GT_NOP: - case GT_LABEL: - case GT_SWIFT_ERROR: - return true; - - default: - break; - } - - return false; - } - - /* Is it a 'simple' unary/binary operator? */ - - if (kind & GTK_UNOP) - { - if (oper == GT_IND) - { - if (op1->TypeIs(TYP_STRUCT)) - { - // Rare case -- need contextual information to check equality in some cases. - return false; - } - - if ((op1->gtFlags & GTF_IND_FLAGS) != (op2->gtFlags & GTF_IND_FLAGS)) - { - return false; - } - } - - if (IsExOp(kind)) - { - // ExOp operators extend unary operator with extra, non-GenTree* members. In many cases, - // these should be included in the comparison. - switch (oper) - { - case GT_STORE_LCL_FLD: - if ((op1->AsLclFld()->GetLclOffs() != op2->AsLclFld()->GetLclOffs()) || - (op1->AsLclFld()->GetLayout() != op2->AsLclFld()->GetLayout())) - { - return false; - } - FALLTHROUGH; - case GT_STORE_LCL_VAR: - if (op1->AsLclVarCommon()->GetLclNum() != op2->AsLclVarCommon()->GetLclNum()) - { - return false; - } - break; - - case GT_ARR_LENGTH: - if (op1->AsArrLen()->ArrLenOffset() != op2->AsArrLen()->ArrLenOffset()) - { - return false; - } - break; - case GT_MDARR_LENGTH: - case GT_MDARR_LOWER_BOUND: - if ((op1->AsMDArr()->Dim() != op2->AsMDArr()->Dim()) || - (op1->AsMDArr()->Rank() != op2->AsMDArr()->Rank())) - { - return false; - } - break; - case GT_CAST: - if (op1->AsCast()->gtCastType != op2->AsCast()->gtCastType) - { - return false; - } - break; - - case GT_BLK: - if (op1->AsBlk()->GetLayout() != op2->AsBlk()->GetLayout()) - { - return false; - } - - if ((op1->gtFlags & GTF_IND_FLAGS) != (op2->gtFlags & GTF_IND_FLAGS)) - { - return false; - } - - break; - - case GT_FIELD_ADDR: - if (op1->AsFieldAddr()->gtFldHnd != op2->AsFieldAddr()->gtFldHnd) - { - return false; - } - break; - - // For the ones below no extra argument matters for comparison. - case GT_BOX: - case GT_RUNTIMELOOKUP: - case GT_ARR_ADDR: - break; - - default: - assert(!"unexpected unary ExOp operator"); - } - } - return Compare(op1->AsOp()->gtOp1, op2->AsOp()->gtOp1); - } - - if (kind & GTK_BINOP) - { - if (IsExOp(kind)) - { - // ExOp operators extend unary operator with extra, non-GenTree* members. In many cases, - // these should be included in the hash code. - switch (oper) - { - case GT_STORE_BLK: - if (op1->AsBlk()->GetLayout() != op2->AsBlk()->GetLayout()) - { - return false; - } - FALLTHROUGH; - - case GT_STOREIND: - if ((op1->gtFlags & GTF_IND_FLAGS) != (op2->gtFlags & GTF_IND_FLAGS)) - { - return false; - } - break; - - case GT_INTRINSIC: - if (op1->AsIntrinsic()->gtIntrinsicName != op2->AsIntrinsic()->gtIntrinsicName) - { - return false; - } - break; - case GT_LEA: - if (op1->AsAddrMode()->gtScale != op2->AsAddrMode()->gtScale) - { - return false; - } - if (op1->AsAddrMode()->Offset() != op2->AsAddrMode()->Offset()) - { - return false; - } - break; - case GT_BOUNDS_CHECK: - if (op1->AsBoundsChk()->gtThrowKind != op2->AsBoundsChk()->gtThrowKind) - { - return false; - } - break; - case GT_INDEX_ADDR: - if (op1->AsIndexAddr()->gtElemSize != op2->AsIndexAddr()->gtElemSize) - { - return false; - } - break; - - // For the ones below no extra argument matters for comparison. - case GT_QMARK: - break; - - default: - assert(!"unexpected binary ExOp operator"); - } - } - - if (op1->AsOp()->gtOp2) - { - if (!Compare(op1->AsOp()->gtOp1, op2->AsOp()->gtOp1, swapOK)) - { - if (swapOK && OperIsCommutative(oper) && - ((op1->AsOp()->gtOp1->gtFlags | op1->AsOp()->gtOp2->gtFlags | op2->AsOp()->gtOp1->gtFlags | - op2->AsOp()->gtOp2->gtFlags) & - GTF_ALL_EFFECT) == 0) - { - if (Compare(op1->AsOp()->gtOp1, op2->AsOp()->gtOp2, swapOK)) - { - op1 = op1->AsOp()->gtOp2; - op2 = op2->AsOp()->gtOp1; - goto AGAIN; - } - } - - return false; - } - - op1 = op1->AsOp()->gtOp2; - op2 = op2->AsOp()->gtOp2; - - goto AGAIN; - } - else - { - - op1 = op1->AsOp()->gtOp1; - op2 = op2->AsOp()->gtOp1; - - if (!op1) - { - return (op2 == nullptr); - } - if (!op2) - { - return false; - } - - goto AGAIN; - } - } - - /* See what kind of a special operator we have here */ - - switch (oper) - { - case GT_CALL: - return GenTreeCall::Equals(op1->AsCall(), op2->AsCall()); - -#ifdef FEATURE_HW_INTRINSICS - case GT_HWINTRINSIC: - return GenTreeHWIntrinsic::Equals(op1->AsHWIntrinsic(), op2->AsHWIntrinsic()); -#endif - - case GT_ARR_ELEM: - - if (op1->AsArrElem()->gtArrRank != op2->AsArrElem()->gtArrRank) - { - return false; - } - - // NOTE: gtArrElemSize may need to be handled - - unsigned dim; - for (dim = 0; dim < op1->AsArrElem()->gtArrRank; dim++) - { - if (!Compare(op1->AsArrElem()->gtArrInds[dim], op2->AsArrElem()->gtArrInds[dim])) - { - return false; - } - } - - op1 = op1->AsArrElem()->gtArrObj; - op2 = op2->AsArrElem()->gtArrObj; - goto AGAIN; - - case GT_PHI: - return GenTreePhi::Equals(op1->AsPhi(), op2->AsPhi()); - - case GT_FIELD_LIST: - return GenTreeFieldList::Equals(op1->AsFieldList(), op2->AsFieldList()); - - case GT_CMPXCHG: - return Compare(op1->AsCmpXchg()->Addr(), op2->AsCmpXchg()->Addr()) && - Compare(op1->AsCmpXchg()->Data(), op2->AsCmpXchg()->Data()) && - Compare(op1->AsCmpXchg()->Comparand(), op2->AsCmpXchg()->Comparand()); - - default: - assert(!"unexpected operator"); - } - - return false; -} - -//------------------------------------------------------------------------ -// gtHasRef: Find out whether the given tree contains a local. -// -// Arguments: -// tree - tree to find the local in -// lclNum - the local's number -// -// Return Value: -// Whether "tree" has any local nodes that refer to the local. -// -/* static */ bool Compiler::gtHasRef(GenTree* tree, unsigned lclNum) -{ - if (tree == nullptr) - { - return false; - } - - if (tree->OperIsLeaf()) - { - if (tree->OperIsAnyLocal() && (tree->AsLclVarCommon()->GetLclNum() == lclNum)) - { - return true; - } - if (tree->OperIs(GT_RET_EXPR)) - { - return gtHasRef(tree->AsRetExpr()->gtInlineCandidate, lclNum); - } - - return false; - } - - if (tree->OperIsUnary()) - { - if (tree->OperIsLocalStore() && (tree->AsLclVarCommon()->GetLclNum() == lclNum)) - { - return true; - } - - return gtHasRef(tree->AsUnOp()->gtGetOp1(), lclNum); - } - - if (tree->OperIsBinary()) - { - return gtHasRef(tree->AsOp()->gtGetOp1(), lclNum) || gtHasRef(tree->AsOp()->gtGetOp2(), lclNum); - } - - bool result = false; - tree->VisitOperands([lclNum, &result](GenTree* operand) -> GenTree::VisitResult { - if (gtHasRef(operand, lclNum)) - { - result = true; - return GenTree::VisitResult::Abort; - } - - return GenTree::VisitResult::Continue; - }); - - return result; -} - -//------------------------------------------------------------------------------ -// gtHasLocalsWithAddrOp: -// Check if this tree contains locals with lvHasLdAddrOp or -// IsAddressExposed() flags set. Does a full tree walk. -// -// Paramters: -// tree - the tree -// -// Return Value: -// True if any sub tree is such a local. -// -bool Compiler::gtHasLocalsWithAddrOp(GenTree* tree) -{ - struct LocalsWithAddrOpVisitor : GenTreeVisitor - { - enum - { - DoPreOrder = true, - DoLclVarsOnly = true, - }; - - LocalsWithAddrOpVisitor(Compiler* comp) - : GenTreeVisitor(comp) - { - } - - fgWalkResult PreOrderVisit(GenTree** use, GenTree* user) - { - LclVarDsc* varDsc = m_compiler->lvaGetDesc((*use)->AsLclVarCommon()); - if (varDsc->lvHasLdAddrOp || varDsc->IsAddressExposed()) - { - return WALK_ABORT; - } - - return WALK_CONTINUE; - } - }; - - LocalsWithAddrOpVisitor visitor(this); - return visitor.WalkTree(&tree, nullptr) == WALK_ABORT; -} - -//------------------------------------------------------------------------------ -// gtHasAddressExposedLocal: -// Check if this tree contains locals with IsAddressExposed() flags set. Does -// a full tree walk. -// -// Paramters: -// tree - the tree -// -// Return Value: -// True if any sub tree is such a local. -// -bool Compiler::gtHasAddressExposedLocals(GenTree* tree) -{ - struct Visitor : GenTreeVisitor - { - enum - { - DoPreOrder = true, - DoLclVarsOnly = true, - }; - - Visitor(Compiler* comp) - : GenTreeVisitor(comp) - { - } - - fgWalkResult PreOrderVisit(GenTree** use, GenTree* user) - { - LclVarDsc* varDsc = m_compiler->lvaGetDesc((*use)->AsLclVarCommon()); - if (varDsc->IsAddressExposed()) - { - return WALK_ABORT; - } - - return WALK_CONTINUE; - } - }; - - Visitor visitor(this); - return visitor.WalkTree(&tree, nullptr) == WALK_ABORT; -} - -#ifdef DEBUG - -/***************************************************************************** - * - * Helper used to compute hash values for trees. - */ - -inline unsigned genTreeHashAdd(unsigned old, unsigned add) -{ - return (old + old / 2) ^ add; -} - -inline unsigned genTreeHashAdd(unsigned old, void* add) -{ - return genTreeHashAdd(old, (unsigned)(size_t)add); -} - -/***************************************************************************** - * - * Given an arbitrary expression tree, compute a hash value for it. - */ - -unsigned Compiler::gtHashValue(GenTree* tree) -{ - genTreeOps oper; - unsigned kind; - - unsigned hash = 0; - - GenTree* temp; - -AGAIN: - assert(tree); - - /* Figure out what kind of a node we have */ - - oper = tree->OperGet(); - kind = tree->OperKind(); - - /* Include the operator value in the hash */ - - hash = genTreeHashAdd(hash, oper); - - /* Is this a leaf node? */ - - if (kind & GTK_LEAF) - { - size_t add; - - switch (oper) - { - UINT64 bits; - case GT_LCL_VAR: - add = tree->AsLclVar()->GetLclNum(); - break; - case GT_LCL_FLD: - hash = genTreeHashAdd(hash, tree->AsLclFld()->GetLclNum()); - hash = genTreeHashAdd(hash, tree->AsLclFld()->GetLayout()); - add = tree->AsLclFld()->GetLclOffs(); - break; - - case GT_CNS_INT: - add = tree->AsIntCon()->gtIconVal; - break; - case GT_CNS_LNG: - bits = (UINT64)tree->AsLngCon()->gtLconVal; -#ifdef HOST_64BIT - add = bits; -#else // 32-bit host - add = genTreeHashAdd(uhi32(bits), ulo32(bits)); -#endif - break; - case GT_CNS_DBL: - { - double dcon = tree->AsDblCon()->DconValue(); - memcpy(&bits, &dcon, sizeof(dcon)); -#ifdef HOST_64BIT - add = bits; -#else // 32-bit host - add = genTreeHashAdd(uhi32(bits), ulo32(bits)); -#endif - break; - } - case GT_CNS_STR: - add = tree->AsStrCon()->gtSconCPX; - break; - -#if defined(FEATURE_SIMD) - case GT_CNS_VEC: - { - GenTreeVecCon* vecCon = tree->AsVecCon(); - add = 0; - - switch (vecCon->TypeGet()) - { -#if defined(TARGET_XARCH) - case TYP_SIMD64: - { - add = genTreeHashAdd(ulo32(add), vecCon->gtSimdVal.u32[15]); - add = genTreeHashAdd(ulo32(add), vecCon->gtSimdVal.u32[14]); - add = genTreeHashAdd(ulo32(add), vecCon->gtSimdVal.u32[13]); - add = genTreeHashAdd(ulo32(add), vecCon->gtSimdVal.u32[12]); - add = genTreeHashAdd(ulo32(add), vecCon->gtSimdVal.u32[11]); - add = genTreeHashAdd(ulo32(add), vecCon->gtSimdVal.u32[10]); - add = genTreeHashAdd(ulo32(add), vecCon->gtSimdVal.u32[9]); - add = genTreeHashAdd(ulo32(add), vecCon->gtSimdVal.u32[8]); - FALLTHROUGH; - } - - case TYP_SIMD32: - { - add = genTreeHashAdd(ulo32(add), vecCon->gtSimdVal.u32[7]); - add = genTreeHashAdd(ulo32(add), vecCon->gtSimdVal.u32[6]); - add = genTreeHashAdd(ulo32(add), vecCon->gtSimdVal.u32[5]); - add = genTreeHashAdd(ulo32(add), vecCon->gtSimdVal.u32[4]); - FALLTHROUGH; - } -#endif // TARGET_XARCH - - case TYP_SIMD16: - { - add = genTreeHashAdd(ulo32(add), vecCon->gtSimdVal.u32[3]); - FALLTHROUGH; - } - - case TYP_SIMD12: - { - add = genTreeHashAdd(ulo32(add), vecCon->gtSimdVal.u32[2]); - FALLTHROUGH; - } - - case TYP_SIMD8: - { - add = genTreeHashAdd(ulo32(add), vecCon->gtSimdVal.u32[1]); - add = genTreeHashAdd(ulo32(add), vecCon->gtSimdVal.u32[0]); - break; - } - - default: - { - unreached(); - } - } - break; - } -#endif // FEATURE_SIMD - -#if defined(FEATURE_MASKED_HW_INTRINSICS) - case GT_CNS_MSK: - { - GenTreeMskCon* mskCon = tree->AsMskCon(); - add = 0; - - add = genTreeHashAdd(ulo32(add), mskCon->gtSimdMaskVal.u32[1]); - add = genTreeHashAdd(ulo32(add), mskCon->gtSimdMaskVal.u32[0]); - break; - } -#endif // FEATURE_MASKED_HW_INTRINSICS - - case GT_JMP: - add = tree->AsVal()->gtVal1; - break; - - default: - add = 0; - break; - } - - // clang-format off - // narrow 'add' into a 32-bit 'val' - unsigned val; -#ifdef HOST_64BIT - val = genTreeHashAdd(uhi32(add), ulo32(add)); -#else // 32-bit host - val = add; -#endif - // clang-format on - - hash = genTreeHashAdd(hash, val); - goto DONE; - } - - /* Is it a 'simple' unary/binary operator? */ - - GenTree* op1; - - if (kind & GTK_UNOP) - { - op1 = tree->AsOp()->gtOp1; - /* Special case: no sub-operand at all */ - - if (GenTree::IsExOp(kind)) - { - // ExOp operators extend operators with extra, non-GenTree* members. In many cases, - // these should be included in the hash code. - switch (oper) - { - case GT_STORE_LCL_VAR: - hash = genTreeHashAdd(hash, tree->AsLclVar()->GetLclNum()); - break; - case GT_STORE_LCL_FLD: - hash = genTreeHashAdd(hash, tree->AsLclFld()->GetLclNum()); - hash = genTreeHashAdd(hash, tree->AsLclFld()->GetLclOffs()); - hash = genTreeHashAdd(hash, tree->AsLclFld()->GetLayout()); - break; - case GT_STOREIND: - hash = genTreeHashAdd(hash, tree->AsStoreInd()->GetRMWStatus()); - break; - case GT_ARR_LENGTH: - hash += tree->AsArrLen()->ArrLenOffset(); - break; - case GT_MDARR_LENGTH: - case GT_MDARR_LOWER_BOUND: - hash += tree->AsMDArr()->Dim(); - hash += tree->AsMDArr()->Rank(); - break; - case GT_CAST: - hash ^= tree->AsCast()->gtCastType; - break; - case GT_INDEX_ADDR: - hash += tree->AsIndexAddr()->gtElemSize; - break; - case GT_ALLOCOBJ: - hash = genTreeHashAdd(hash, static_cast( - reinterpret_cast(tree->AsAllocObj()->gtAllocObjClsHnd))); - hash = genTreeHashAdd(hash, tree->AsAllocObj()->gtNewHelper); - break; - case GT_RUNTIMELOOKUP: - hash = genTreeHashAdd(hash, static_cast( - reinterpret_cast(tree->AsRuntimeLookup()->gtHnd))); - break; - case GT_BLK: - hash = - genTreeHashAdd(hash, - static_cast(reinterpret_cast(tree->AsBlk()->GetLayout()))); - break; - - case GT_FIELD_ADDR: - hash = genTreeHashAdd(hash, tree->AsFieldAddr()->gtFldHnd); - break; - - // For the ones below no extra argument matters for comparison. - case GT_BOX: - case GT_ARR_ADDR: - break; - - default: - assert(!"unexpected unary ExOp operator"); - } - } - - if (!op1) - { - goto DONE; - } - - tree = op1; - goto AGAIN; - } - - if (kind & GTK_BINOP) - { - if (GenTree::IsExOp(kind)) - { - // ExOp operators extend operators with extra, non-GenTree* members. In many cases, - // these should be included in the hash code. - switch (oper) - { - case GT_STOREIND: - hash = genTreeHashAdd(hash, tree->AsStoreInd()->GetRMWStatus()); - break; - - case GT_STORE_BLK: - hash = genTreeHashAdd(hash, tree->AsBlk()->GetLayout()); - break; - - case GT_INTRINSIC: - hash += tree->AsIntrinsic()->gtIntrinsicName; - break; - case GT_LEA: - hash += static_cast(tree->AsAddrMode()->Offset() << 3) + tree->AsAddrMode()->gtScale; - break; - - case GT_BOUNDS_CHECK: - hash = genTreeHashAdd(hash, tree->AsBoundsChk()->gtThrowKind); - break; - - // For the ones below no extra argument matters for comparison. - case GT_QMARK: - case GT_INDEX_ADDR: - break; - -#ifdef FEATURE_HW_INTRINSICS - case GT_HWINTRINSIC: - hash += tree->AsHWIntrinsic()->GetHWIntrinsicId(); - hash += tree->AsHWIntrinsic()->GetSimdBaseType(); - hash += tree->AsHWIntrinsic()->GetSimdSize(); - hash += tree->AsHWIntrinsic()->GetAuxiliaryType(); - hash += tree->AsHWIntrinsic()->GetRegByIndex(1); - break; -#endif // FEATURE_HW_INTRINSICS - - default: - assert(!"unexpected binary ExOp operator"); - } - } - - op1 = tree->AsOp()->gtOp1; - GenTree* op2 = tree->AsOp()->gtOp2; - - /* Is there a second sub-operand? */ - - if (!op2) - { - /* Special case: no sub-operands at all */ - - if (!op1) - { - goto DONE; - } - - /* This is a unary operator */ - - tree = op1; - goto AGAIN; - } - - /* This is a binary operator */ - - unsigned hsh1 = gtHashValue(op1); - - /* Add op1's hash to the running value and continue with op2 */ - - hash = genTreeHashAdd(hash, hsh1); - - tree = op2; - goto AGAIN; - } - - /* See what kind of a special operator we have here */ - switch (tree->gtOper) - { - case GT_ARR_ELEM: - - hash = genTreeHashAdd(hash, gtHashValue(tree->AsArrElem()->gtArrObj)); - - unsigned dim; - for (dim = 0; dim < tree->AsArrElem()->gtArrRank; dim++) - { - hash = genTreeHashAdd(hash, gtHashValue(tree->AsArrElem()->gtArrInds[dim])); - } - - break; - - case GT_CALL: - for (CallArg& arg : tree->AsCall()->gtArgs.Args()) - { - if (arg.GetEarlyNode() != nullptr) - { - hash = genTreeHashAdd(hash, gtHashValue(arg.GetEarlyNode())); - } - - if (arg.GetLateNode() != nullptr) - { - hash = genTreeHashAdd(hash, gtHashValue(arg.GetLateNode())); - } - } - - if (tree->AsCall()->gtCallType == CT_INDIRECT) - { - temp = tree->AsCall()->gtCallAddr; - assert(temp); - hash = genTreeHashAdd(hash, gtHashValue(temp)); - } - else - { - hash = genTreeHashAdd(hash, tree->AsCall()->gtCallMethHnd); - } - - break; - -#if defined(FEATURE_HW_INTRINSICS) - case GT_HWINTRINSIC: - // TODO-List: rewrite with a general visitor / iterator? - for (GenTree* operand : tree->AsMultiOp()->Operands()) - { - hash = genTreeHashAdd(hash, gtHashValue(operand)); - } - break; -#endif // FEATURE_HW_INTRINSICS - - case GT_PHI: - for (GenTreePhi::Use& use : tree->AsPhi()->Uses()) - { - hash = genTreeHashAdd(hash, gtHashValue(use.GetNode())); - } - break; - - case GT_FIELD_LIST: - for (GenTreeFieldList::Use& use : tree->AsFieldList()->Uses()) - { - hash = genTreeHashAdd(hash, gtHashValue(use.GetNode())); - } - break; - - case GT_CMPXCHG: - hash = genTreeHashAdd(hash, gtHashValue(tree->AsCmpXchg()->Addr())); - hash = genTreeHashAdd(hash, gtHashValue(tree->AsCmpXchg()->Data())); - hash = genTreeHashAdd(hash, gtHashValue(tree->AsCmpXchg()->Comparand())); - break; - - default: -#ifdef DEBUG - gtDispTree(tree); -#endif - assert(!"unexpected operator"); - break; - } - -DONE: - - return hash; -} - -#endif // DEBUG - -/***************************************************************************** - * - * Return a relational operator that is the reverse of the given one. - */ - -/* static */ -genTreeOps GenTree::ReverseRelop(genTreeOps relop) -{ - static constexpr genTreeOps reverseOps[] = { - GT_NE, // GT_EQ - GT_EQ, // GT_NE - GT_GE, // GT_LT - GT_GT, // GT_LE - GT_LT, // GT_GE - GT_LE, // GT_GT - GT_TEST_NE, // GT_TEST_EQ - GT_TEST_EQ, // GT_TEST_NE -#ifdef TARGET_XARCH - GT_BITTEST_NE, // GT_BITTEST_EQ - GT_BITTEST_EQ, // GT_BITTEST_NE -#endif - }; - - static_assert_no_msg(reverseOps[GT_EQ - GT_EQ] == GT_NE); - static_assert_no_msg(reverseOps[GT_NE - GT_EQ] == GT_EQ); - static_assert_no_msg(reverseOps[GT_LT - GT_EQ] == GT_GE); - static_assert_no_msg(reverseOps[GT_LE - GT_EQ] == GT_GT); - static_assert_no_msg(reverseOps[GT_GE - GT_EQ] == GT_LT); - static_assert_no_msg(reverseOps[GT_GT - GT_EQ] == GT_LE); - static_assert_no_msg(reverseOps[GT_TEST_EQ - GT_EQ] == GT_TEST_NE); - static_assert_no_msg(reverseOps[GT_TEST_NE - GT_EQ] == GT_TEST_EQ); -#ifdef TARGET_XARCH - static_assert_no_msg(reverseOps[GT_BITTEST_EQ - GT_EQ] == GT_BITTEST_NE); - static_assert_no_msg(reverseOps[GT_BITTEST_NE - GT_EQ] == GT_BITTEST_EQ); -#endif - - assert(OperIsCompare(relop)); - assert(relop >= GT_EQ && (unsigned)(relop - GT_EQ) < sizeof(reverseOps)); - - return reverseOps[relop - GT_EQ]; -} - -/***************************************************************************** - * - * Return a relational operator that will work for swapped operands. - */ - -/* static */ -genTreeOps GenTree::SwapRelop(genTreeOps relop) -{ - static constexpr genTreeOps swapOps[] = { - GT_EQ, // GT_EQ - GT_NE, // GT_NE - GT_GT, // GT_LT - GT_GE, // GT_LE - GT_LE, // GT_GE - GT_LT, // GT_GT - GT_TEST_EQ, // GT_TEST_EQ - GT_TEST_NE, // GT_TEST_NE -#ifdef TARGET_XARCH - GT_BITTEST_EQ, // GT_BITTEST_EQ - GT_BITTEST_NE, // GT_BITTEST_NE -#endif - }; - - static_assert_no_msg(swapOps[GT_EQ - GT_EQ] == GT_EQ); - static_assert_no_msg(swapOps[GT_NE - GT_EQ] == GT_NE); - static_assert_no_msg(swapOps[GT_LT - GT_EQ] == GT_GT); - static_assert_no_msg(swapOps[GT_LE - GT_EQ] == GT_GE); - static_assert_no_msg(swapOps[GT_GE - GT_EQ] == GT_LE); - static_assert_no_msg(swapOps[GT_GT - GT_EQ] == GT_LT); - static_assert_no_msg(swapOps[GT_TEST_EQ - GT_EQ] == GT_TEST_EQ); - static_assert_no_msg(swapOps[GT_TEST_NE - GT_EQ] == GT_TEST_NE); -#ifdef TARGET_XARCH - static_assert_no_msg(swapOps[GT_BITTEST_EQ - GT_EQ] == GT_BITTEST_EQ); - static_assert_no_msg(swapOps[GT_BITTEST_NE - GT_EQ] == GT_BITTEST_NE); -#endif - - assert(OperIsCompare(relop)); - assert(relop >= GT_EQ && (unsigned)(relop - GT_EQ) < sizeof(swapOps)); - - return swapOps[relop - GT_EQ]; -} - -/***************************************************************************** - * - * Reverse the meaning of the given test condition. - */ - -GenTree* Compiler::gtReverseCond(GenTree* tree) -{ - if (tree->OperIsCompare()) - { - tree->SetOper(GenTree::ReverseRelop(tree->OperGet())); - - // Flip the GTF_RELOP_NAN_UN bit - // a ord b === (a != NaN && b != NaN) - // a unord b === (a == NaN || b == NaN) - // => !(a ord b) === (a unord b) - if (varTypeIsFloating(tree->AsOp()->gtOp1->TypeGet())) - { - tree->gtFlags ^= GTF_RELOP_NAN_UN; - } - } - else if (tree->OperIs(GT_JCC, GT_SETCC)) - { - GenTreeCC* cc = tree->AsCC(); - cc->gtCondition = GenCondition::Reverse(cc->gtCondition); - } - else if (tree->OperIs(GT_JCMP, GT_JTEST)) - { - GenTreeOpCC* opCC = tree->AsOpCC(); - opCC->gtCondition = GenCondition::Reverse(opCC->gtCondition); - } - else - { - tree = gtNewOperNode(GT_NOT, TYP_INT, tree); - } - - return tree; -} - -#if !defined(TARGET_64BIT) || defined(TARGET_ARM64) -//------------------------------------------------------------------------------ -// IsValidLongMul : Check for long multiplication with 32 bit operands. -// -// Recognizes the following tree: MUL(CAST(long <- int), CAST(long <- int) or CONST), -// where CONST must be an integer constant that fits in 32 bits. Will try to detect -// cases when the multiplication cannot overflow and return "true" for them. -// -// This function does not change the state of the tree and is usable in LIR. -// -// Return Value: -// Whether this GT_MUL tree is a valid long multiplication candidate. -// -bool GenTreeOp::IsValidLongMul() -{ - assert(OperIs(GT_MUL)); - - GenTree* op1 = gtGetOp1(); - GenTree* op2 = gtGetOp2(); - - if (!TypeIs(TYP_LONG)) - { - return false; - } - - assert(op1->TypeIs(TYP_LONG)); - assert(op2->TypeIs(TYP_LONG)); - - if (!(op1->OperIs(GT_CAST) && genActualTypeIsInt(op1->AsCast()->CastOp()))) - { - return false; - } - - if (!(op2->OperIs(GT_CAST) && genActualTypeIsInt(op2->AsCast()->CastOp())) && - !(op2->IsIntegralConst() && FitsIn(op2->AsIntConCommon()->IntegralValue()))) - { - return false; - } - - if (op1->gtOverflow() || op2->gtOverflowEx()) - { - return false; - } - - if (gtOverflow()) - { - auto getMaxValue = [this](GenTree* op) -> int64_t { - if (op->OperIs(GT_CAST)) - { - if (op->IsUnsigned()) - { - switch (op->AsCast()->CastOp()->TypeGet()) - { - case TYP_UBYTE: - return UINT8_MAX; - case TYP_USHORT: - return UINT16_MAX; - default: - return UINT32_MAX; - } - } - - return IsUnsigned() ? static_cast(UINT64_MAX) : INT32_MIN; - } - - return op->AsIntConCommon()->IntegralValue(); - }; - - int64_t maxOp1 = getMaxValue(op1); - int64_t maxOp2 = getMaxValue(op2); - - if (CheckedOps::MulOverflows(maxOp1, maxOp2, IsUnsigned())) - { - return false; - } - } - - // Both operands must extend the same way. - bool op1ZeroExtends = op1->IsUnsigned(); - bool op2ZeroExtends = op2->OperIs(GT_CAST) ? op2->IsUnsigned() : op2->AsIntConCommon()->IntegralValue() >= 0; - bool op2AnyExtensionIsSuitable = op2->IsIntegralConst() && op2ZeroExtends; - if ((op1ZeroExtends != op2ZeroExtends) && !op2AnyExtensionIsSuitable) - { - return false; - } - - return true; -} - -#if !defined(TARGET_64BIT) && defined(DEBUG) -//------------------------------------------------------------------------------ -// DebugCheckLongMul : Checks that a GTF_MUL_64RSLT tree is a valid MUL_LONG. -// -// Notes: -// This function is defined for 32 bit targets only because we *must* maintain -// the MUL_LONG-compatible tree shape throughout the compilation from morph to -// decomposition, since we do not have (great) ability to create new calls in LIR. -// -// It is for this reason that we recognize MUL_LONGs early in morph, mark them with -// a flag and then pessimize various places (e. g. assertion propagation) to not look -// at them. In contrast, on ARM64 we recognize MUL_LONGs late, in lowering, and thus -// do not need this function. -// -void GenTreeOp::DebugCheckLongMul() -{ - assert(OperIs(GT_MUL)); - assert(Is64RsltMul()); - assert(TypeIs(TYP_LONG)); - assert(!gtOverflow()); - - GenTree* op1 = gtGetOp1(); - GenTree* op2 = gtGetOp2(); - - assert(op1->TypeIs(TYP_LONG)); - assert(op2->TypeIs(TYP_LONG)); - - // op1 has to be CAST(long <- int) - assert(op1->OperIs(GT_CAST) && genActualTypeIsInt(op1->AsCast()->CastOp())); - assert(!op1->gtOverflow()); - - // op2 has to be CAST(long <- int) or a suitably small constant. - assert((op2->OperIs(GT_CAST) && genActualTypeIsInt(op2->AsCast()->CastOp())) || - (op2->IsIntegralConst() && FitsIn(op2->AsIntConCommon()->IntegralValue()))); - assert(!op2->gtOverflowEx()); - - // Both operands must extend the same way. - bool op1ZeroExtends = op1->IsUnsigned(); - bool op2ZeroExtends = op2->OperIs(GT_CAST) ? op2->IsUnsigned() : op2->AsIntConCommon()->IntegralValue() >= 0; - bool op2AnyExtensionIsSuitable = op2->IsIntegralConst() && op2ZeroExtends; - assert((op1ZeroExtends == op2ZeroExtends) || op2AnyExtensionIsSuitable); - - // Do unsigned mul iff both operands are zero-extending. - assert(op1->IsUnsigned() == IsUnsigned()); -} -#endif // !defined(TARGET_64BIT) && defined(DEBUG) -#endif // !defined(TARGET_64BIT) || defined(TARGET_ARM64) - -unsigned Compiler::gtSetCallArgsOrder(CallArgs* args, bool lateArgs, int* callCostEx, int* callCostSz) -{ - unsigned level = 0; - unsigned costEx = 0; - unsigned costSz = 0; - - auto update = [&level, &costEx, &costSz, lateArgs](GenTree* argNode, unsigned argLevel) { - if (argLevel > level) - { - level = argLevel; - } - - if (argNode->GetCostEx() != 0) - { - costEx += argNode->GetCostEx(); - costEx += lateArgs ? 0 : IND_COST_EX; - } - if (argNode->GetCostSz() != 0) - { - costSz += argNode->GetCostSz(); -#ifdef TARGET_XARCH - if (lateArgs) // push is smaller than mov to reg -#endif - { - costSz += 1; - } - } - }; - - if (lateArgs) - { - for (CallArg& arg : args->LateArgs()) - { - GenTree* node = arg.GetLateNode(); - unsigned level = gtSetEvalOrder(node); - update(node, level); - } - } - else - { - for (CallArg& arg : args->EarlyArgs()) - { - GenTree* node = arg.GetEarlyNode(); - unsigned level = gtSetEvalOrder(node); - update(node, level); - } - } - - *callCostEx += costEx; - *callCostSz += costSz; - - return level; -} - -#if defined(FEATURE_SIMD) || defined(FEATURE_HW_INTRINSICS) -//------------------------------------------------------------------------ -// gtSetMultiOpOrder: Calculate the costs for a MultiOp. -// -// Arguments: -// multiOp - The MultiOp tree in question -// -// Return Value: -// The Sethi "complexity" for this tree (the idealized number of -// registers needed to evaluate it). -// -unsigned Compiler::gtSetMultiOpOrder(GenTreeMultiOp* multiOp) -{ - // Most HWI nodes are simple arithmetic operations. - // - int costEx = 1; - int costSz = 1; - unsigned level = 0; - - bool optsEnabled = opts.OptimizationEnabled(); - -#if defined(FEATURE_HW_INTRINSICS) - if (multiOp->OperIs(GT_HWINTRINSIC) && optsEnabled) - { - GenTreeHWIntrinsic* hwTree = multiOp->AsHWIntrinsic(); -#if defined(TARGET_XARCH) - if ((hwTree->GetOperandCount() == 1) && hwTree->OperIsMemoryLoadOrStore()) - { - costEx = IND_COST_EX; - costSz = 2; - - GenTree* const addrNode = hwTree->Op(1); - level = gtSetEvalOrder(addrNode); - GenTree* const addr = addrNode->gtEffectiveVal(); - - // See if we can form a complex addressing mode. - if (addr->OperIs(GT_ADD) && gtMarkAddrMode(addr, &costEx, &costSz, hwTree->TypeGet())) - { - // Nothing to do, costs have been set. - } - else - { - costEx += addr->GetCostEx(); - costSz += addr->GetCostSz(); - } - - hwTree->SetCosts(costEx, costSz); - return level; - } -#endif - switch (hwTree->GetHWIntrinsicId()) - { - case NI_Vector128_Create: - case NI_Vector128_CreateScalar: - case NI_Vector128_CreateScalarUnsafe: -#if defined(TARGET_XARCH) - case NI_Vector256_Create: - case NI_Vector256_CreateScalar: - case NI_Vector256_CreateScalarUnsafe: - case NI_Vector512_Create: - case NI_Vector512_CreateScalar: - case NI_Vector512_CreateScalarUnsafe: -#elif defined(TARGET_ARM64) - case NI_Vector64_Create: - case NI_Vector64_CreateScalar: - case NI_Vector64_CreateScalarUnsafe: -#endif - { - if ((hwTree->GetOperandCount() == 1) && hwTree->Op(1)->OperIsConst()) - { - // Vector.Create(cns) is cheap but not that cheap to be (1,1) - costEx = IND_COST_EX; - costSz = 2; - level = gtSetEvalOrder(hwTree->Op(1)); - hwTree->SetCosts(costEx, costSz); - return level; - } - break; - } - default: - break; - } - } -#endif // defined(FEATURE_SIMD) || defined(FEATURE_HW_INTRINSICS) - - // The binary case is special because of GTF_REVERSE_OPS. - if (multiOp->GetOperandCount() == 2) - { - unsigned lvl2 = 0; - - // This way we have "level" be the complexity of the - // first tree to be evaluated, and "lvl2" - the second. - if (multiOp->IsReverseOp()) - { - assert(!multiOp->AsHWIntrinsic()->IsUserCall()); - - level = gtSetEvalOrder(multiOp->Op(2)); - lvl2 = gtSetEvalOrder(multiOp->Op(1)); - } - else - { - level = gtSetEvalOrder(multiOp->Op(1)); - lvl2 = gtSetEvalOrder(multiOp->Op(2)); - } - - // We want the more complex tree to be evaluated first. - if (level < lvl2) - { - bool canSwap = false; - - if (!multiOp->AsHWIntrinsic()->IsUserCall()) - { - canSwap = multiOp->IsReverseOp() ? gtCanSwapOrder(multiOp->Op(2), multiOp->Op(1)) - : gtCanSwapOrder(multiOp->Op(1), multiOp->Op(2)); - } - - if (canSwap) - { - assert(!multiOp->AsHWIntrinsic()->IsUserCall()); - - if (multiOp->IsReverseOp()) - { - multiOp->ClearReverseOp(); - } - else - { - multiOp->SetReverseOp(); - } - - std::swap(level, lvl2); - } - } - - if (level < 1) - { - level = lvl2; - } - else if (level == lvl2) - { - level += 1; - } - - if (optsEnabled) - { - // We don't need/have costs in MinOpts - costEx += (multiOp->Op(1)->GetCostEx() + multiOp->Op(2)->GetCostEx()); - costSz += (multiOp->Op(1)->GetCostSz() + multiOp->Op(2)->GetCostSz()); - } - } - else - { - for (size_t i = multiOp->GetOperandCount(); i >= 1; i--) - { - GenTree* op = multiOp->Op(i); - unsigned lvl = gtSetEvalOrder(op); - - level = max(lvl, level + 1); - - if (optsEnabled) - { - // We don't need/have costs in MinOpts - costEx += op->GetCostEx(); - costSz += op->GetCostSz(); - } - } - } - - if (optsEnabled) - { - multiOp->SetCosts(costEx, costSz); - } - return level; -} -#endif - -//----------------------------------------------------------------------------- -// gtWalkOp: Traverse and mark an address expression -// -// Arguments: -// op1WB - An out parameter which is either the address expression, or one -// of its operands. -// op2WB - An out parameter which starts as either null or one of the operands -// of the address expression. -// base - The base address of the addressing mode, or null if 'constOnly' is false -// constOnly - True if we will only traverse into ADDs with constant op2. -// -// This routine is a helper routine for gtSetEvalOrder() and is used to identify the -// base and index nodes, which will be validated against those identified by -// genCreateAddrMode(). -// It also marks the ADD nodes involved in the address expression with the -// GTF_ADDRMODE_NO_CSE flag which prevents them from being considered for CSE's. -// -// Its two output parameters are modified under the following conditions: -// -// It is called once with the original address expression as 'op1WB', and -// with 'constOnly' set to false. On this first invocation, *op1WB is always -// an ADD node, and it will consider the operands of the ADD even if its op2 is -// not a constant. However, when it encounters a non-constant or the base in the -// op2 position, it stops iterating. That operand is returned in the 'op2WB' out -// parameter, and will be considered on the third invocation of this method if -// it is an ADD. -// -// It is called the second time with the two operands of the original expression, in -// the original order, and the third time in reverse order. For these invocations -// 'constOnly' is true, so it will only traverse cascaded ADD nodes if they have a -// constant op2. -// -// The result, after three invocations, is that the values of the two out parameters -// correspond to the base and index in some fashion. This method doesn't attempt -// to determine or validate the scale or offset, if any. -// -// Assumptions (presumed to be ensured by genCreateAddrMode()): -// If an ADD has a constant operand, it is in the op2 position. -// -// Notes: -// This method, and its invocation sequence, are quite confusing, and since they -// were not originally well-documented, this specification is a possibly-imperfect -// reconstruction. -// The motivation for the handling of the NOP case is unclear. -// Note that 'op2WB' is only modified in the initial (!constOnly) case, -// or if a NOP is encountered in the op1 position. -// -void Compiler::gtWalkOp(GenTree** op1WB, GenTree** op2WB, GenTree* base, bool constOnly) -{ - GenTree* op1 = *op1WB; - GenTree* op2 = *op2WB; - - op1 = op1->gtEffectiveVal(); - - // Now we look for op1's with non-overflow GT_ADDs [of constants] - while (op1->OperIs(GT_ADD) && !op1->gtOverflow()) - { - GenTreeOp* add = op1->AsOp(); - GenTree* addOp1 = add->gtGetOp1(); - GenTree* addOp2 = add->gtGetOp2(); - - if (constOnly && (!addOp2->IsCnsIntOrI() || !addOp2->AsIntCon()->ImmedValCanBeFolded(this, GT_ADD))) - { - break; - } - - // Ignore ADD(CNS, CNS-gc-handle) - if (constOnly && addOp2->IsIconHandle(GTF_ICON_OBJ_HDL) && !addOp2->IsIntegralConst(0)) - { - break; - } - - // mark it with GTF_ADDRMODE_NO_CSE - add->gtFlags |= GTF_ADDRMODE_NO_CSE; - - if (!constOnly) - { - op2 = addOp2; - } - op1 = addOp1; - - if (!constOnly && ((op2 == base) || !op2->IsCnsIntOrI() || !op2->AsIntCon()->ImmedValCanBeFolded(this, GT_ADD))) - { - break; - } - - op1 = op1->gtEffectiveVal(); - } - - *op1WB = op1; - *op2WB = op2; -} - -#ifdef DEBUG -/***************************************************************************** - * This is a workaround. It is to help implement an assert in gtSetEvalOrder() that the values - * gtWalkOp() leaves in op1 and op2 correspond with the values of adr, idx, mul, and cns - * that are returned by genCreateAddrMode(). It's essentially impossible to determine - * what gtWalkOp() *should* return for all possible trees. This simply loosens one assert - * to handle the following case: - - indir int - const(h) int 4 field - + byref - lclVar byref V00 this <-- op2 - comma byref <-- adr (base) - indir byte - lclVar byref V00 this - + byref - const int 2 <-- mul == 4 - << int <-- op1 - lclVar int V01 arg1 <-- idx - - * Here, we are planning to generate the address mode [edx+4*eax], where eax = idx and edx = the GT_COMMA expression. - * To check adr equivalence with op2, we need to walk down the GT_ADD tree just like gtWalkOp() does. - */ -GenTree* Compiler::gtWalkOpEffectiveVal(GenTree* op) -{ - while (true) - { - op = op->gtEffectiveVal(); - - if ((op->gtOper != GT_ADD) || op->gtOverflow() || !op->AsOp()->gtOp2->IsCnsIntOrI()) - { - break; - } - - op = op->AsOp()->gtOp1; - } - - return op; -} -#endif // DEBUG - -/***************************************************************************** - * - * Given a tree, set the GetCostEx and GetCostSz() fields which - * are used to measure the relative costs of the codegen of the tree - * - */ - -void Compiler::gtPrepareCost(GenTree* tree) -{ - gtSetEvalOrder(tree); -} - -bool Compiler::gtIsLikelyRegVar(GenTree* tree) -{ - if (!tree->OperIsScalarLocal()) - { - return false; - } - - const LclVarDsc* varDsc = lvaGetDesc(tree->AsLclVar()); - - if (varDsc->lvDoNotEnregister) - { - return false; - } - - // If this is an EH-live var, return false if it is a def, - // as it will have to go to memory. - if (varDsc->lvLiveInOutOfHndlr && ((tree->gtFlags & GTF_VAR_DEF) != 0)) - { - return false; - } - - // Be pessimistic if ref counts are not yet set up. - // - // Perhaps we should be optimistic though. - // See notes in GitHub issue 18969. - if (!lvaLocalVarRefCounted()) - { - return false; - } - - if (varDsc->lvRefCntWtd() < (BB_UNITY_WEIGHT * 3)) - { - return false; - } - -#ifdef TARGET_X86 - if (!varTypeUsesIntReg(tree->TypeGet())) - return false; - if (varTypeIsLong(tree->TypeGet())) - return false; -#endif - - return true; -} - -//------------------------------------------------------------------------ -// gtGetLclVarNodeCost: Calculate the cost for a local variable node. -// -// Used for both uses and defs. Only the node's own cost is calculated. -// -// Arguments: -// node - The node in question -// pCostEx - [out] parameter for the execution cost -// pCostSz - [out] parameter for the size cost -// isLikelyRegVar - Is the local likely to end up enregistered -// -void Compiler::gtGetLclVarNodeCost(GenTreeLclVar* node, int* pCostEx, int* pCostSz, bool isLikelyRegVar) -{ - int costEx; - int costSz; - if (isLikelyRegVar) - { - costEx = 1; - costSz = 1; - /* Sign-extend and zero-extend are more expensive to load */ - if (lvaGetDesc(node)->lvNormalizeOnLoad()) - { - costEx += 1; - costSz += 1; - } - } - else - { - costEx = IND_COST_EX; - costSz = 2; - - // Some types are more expensive to load than others. - if (varTypeIsSmall(node)) - { - costEx += 1; - costSz += 1; - } - else if (node->TypeIs(TYP_STRUCT)) - { - costEx += 2 * IND_COST_EX; - costSz += 2 * 2; - } - } -#if defined(TARGET_AMD64) - // increase costSz for floating point locals - if (varTypeIsFloating(node)) - { - costSz += 1; - if (!isLikelyRegVar) - { - costSz += 1; - } - } -#endif - - *pCostEx = costEx; - *pCostSz = costSz; -} - -//------------------------------------------------------------------------ -// gtGetLclFldNodeCost: Calculate the cost for a local field node. -// -// Used for both uses and defs. Only the node's own cost is calculated. -// -// Arguments: -// node - The node in question -// pCostEx - [out] parameter for the execution cost -// pCostSz - [out] parameter for the size cost -// -void Compiler::gtGetLclFldNodeCost(GenTreeLclFld* node, int* pCostEx, int* pCostSz) -{ - int costEx = IND_COST_EX; - int costSz = 4; - if (varTypeIsSmall(node->TypeGet())) - { - costEx += 1; - costSz += 1; - } - else if (node->TypeIs(TYP_STRUCT)) - { - costEx += 2 * IND_COST_EX; - costSz += 2 * 2; - } - - *pCostEx = costEx; - *pCostSz = costSz; -} - -//------------------------------------------------------------------------ -// gtGetLclFldNodeCost: Calculate the cost for a primitive indir node. -// -// Used for both loads and stores. -// -// Arguments: -// node - The node in question -// pCostEx - [out] parameter for the execution cost -// pCostSz - [out] parameter for the size cost -// -// Return Value: -// Whether the cost calculated includes that of the node's address. -// -bool Compiler::gtGetIndNodeCost(GenTreeIndir* node, int* pCostEx, int* pCostSz) -{ - assert(node->isIndir()); - - // Indirections have a costEx of IND_COST_EX. - int costEx = IND_COST_EX; - int costSz = 2; - bool includesAddrCost = false; - - // If we have to sign-extend or zero-extend, bump the cost. - if (varTypeIsSmall(node)) - { - costEx += 1; - costSz += 1; - } - -#ifdef TARGET_ARM - if (varTypeIsFloating(node)) - { - costSz += 2; - } -#endif // TARGET_ARM - - // Can we form an addressing mode with this indirection? - GenTree* addr = node->Addr(); - - if (addr->gtEffectiveVal()->OperIs(GT_ADD)) - { - // See if we can form a complex addressing mode. - bool doAddrMode = true; -#ifdef TARGET_ARM64 - if (node->IsVolatile()) - { - // For volatile store/loads when address is contained we always emit `dmb` - // if it's not - we emit one-way barriers i.e. ldar/stlr - doAddrMode = false; - } -#endif // TARGET_ARM64 - if (doAddrMode && gtMarkAddrMode(addr, &costEx, &costSz, node->TypeGet())) - { - includesAddrCost = true; - } - } - else if (gtIsLikelyRegVar(addr)) - { - // Indirection of an enregister LCL_VAR, don't increase costEx/costSz. - includesAddrCost = true; - } -#ifdef TARGET_XARCH - else if (addr->IsCnsIntOrI()) - { - // Indirection of a CNS_INT, subtract 1 from costEx makes costEx 3 for x86 and 4 for amd64. - costEx += (addr->GetCostEx() - 1); - costSz += addr->GetCostSz(); - includesAddrCost = true; - } -#endif - - *pCostEx = costEx; - *pCostSz = costSz; - return includesAddrCost; -} - -//------------------------------------------------------------------------ -// gtCanSwapOrder: Returns true iff the secondNode can be swapped with firstNode. -// -// Arguments: -// firstNode - An operand of a tree that can have GTF_REVERSE_OPS set. -// secondNode - The other operand of the tree. -// -// Return Value: -// Returns a boolean indicating whether it is safe to reverse the execution -// order of the two trees, considering any exception, global effects, or -// ordering constraints. -// -bool Compiler::gtCanSwapOrder(GenTree* firstNode, GenTree* secondNode) -{ - bool canSwap = true; - - // Don't swap "CONST_HDL op CNS" - if (firstNode->IsIconHandle() && secondNode->IsIntegralConst()) - { - canSwap = false; - } - - // Relative of order of global / side effects can't be swapped. - - if (optValnumCSE_phase) - { - canSwap = optCSE_canSwap(firstNode, secondNode); - } - - // We cannot swap in the presence of special side effects such as GT_CATCH_ARG. - - if (canSwap && (firstNode->gtFlags & GTF_ORDER_SIDEEFF)) - { - canSwap = false; - } - - // When strict side effect order is disabled we allow GTF_REVERSE_OPS to be set - // when one or both sides contains a GTF_CALL or GTF_EXCEPT. - // Currently only the C and C++ languages allow non strict side effect order. - - unsigned strictEffects = GTF_GLOB_EFFECT; - - if (canSwap && (firstNode->gtFlags & strictEffects)) - { - // op1 has side efects that can't be reordered. - // Check for some special cases where we still may be able to swap. - - if (secondNode->gtFlags & strictEffects) - { - // op2 has also has non reorderable side effects - can't swap. - canSwap = false; - } - else - { - // No side effects in op2 - we can swap iff op1 has no way of modifying op2, - // i.e. through indirect stores or calls or op2 is a constant. - - if (firstNode->gtFlags & strictEffects & GTF_PERSISTENT_SIDE_EFFECTS) - { - // We have to be conservative - can swap iff op2 is constant. - if (!secondNode->IsInvariant()) - { - canSwap = false; - } - } - } - } - return canSwap; -} - -//------------------------------------------------------------------------ -// Given an address expression, compute its costs and addressing mode opportunities, -// and mark addressing mode candidates as GTF_DONT_CSE. -// -// Arguments: -// addr - The address expression -// costEx - The execution cost of this address expression (in/out arg to be updated) -// costEx - The size cost of this address expression (in/out arg to be updated) -// type - The type of the value being referenced by the parent of this address expression. -// -// Return Value: -// Returns true if it finds an addressing mode. -// -// Notes: -// TODO-Throughput - Consider actually instantiating these early, to avoid -// having to re-run the algorithm that looks for them (might also improve CQ). -// -bool Compiler::gtMarkAddrMode(GenTree* addr, int* pCostEx, int* pCostSz, var_types type) -{ - GenTree* addrComma = addr; - addr = addr->gtEffectiveVal(); - - // These are "out" parameters on the call to genCreateAddrMode(): - bool rev; // This will be true if the operands will need to be reversed. At this point we - // don't care about this because we're not yet instantiating this addressing mode. - unsigned mul; // This is the index (scale) value for the addressing mode - ssize_t cns; // This is the constant offset - GenTree* base; // This is the base of the address. - GenTree* idx; // This is the index. - - unsigned naturalMul = 0; -#ifdef TARGET_ARM64 - // Multiplier should be a "natural-scale" power of two number which is equal to target's width. - // - // *(ulong*)(data + index * 8); - can be optimized - // *(ulong*)(data + index * 7); - can not be optimized - // *(int*)(data + index * 2); - can not be optimized - // - naturalMul = genTypeSize(type); -#endif - - if (codeGen->genCreateAddrMode(addr, false /*fold*/, naturalMul, &rev, &base, &idx, &mul, &cns)) - { -#ifdef TARGET_ARM64 - assert((mul == 0) || (mul == 1) || (mul == naturalMul)); -#endif - - // We can form a complex addressing mode, so mark each of the interior - // nodes with GTF_ADDRMODE_NO_CSE and calculate a more accurate cost. - addr->gtFlags |= GTF_ADDRMODE_NO_CSE; - - int originalAddrCostEx = addr->GetCostEx(); - int originalAddrCostSz = addr->GetCostSz(); - int addrModeCostEx = 0; - int addrModeCostSz = 0; - -#ifdef TARGET_XARCH - // addrmodeCount is the count of items that we used to form - // an addressing mode. The maximum value is 4 when we have - // all of these: { base, idx, cns, mul } - // - unsigned addrmodeCount = 0; - if (base) - { - addrModeCostEx += base->GetCostEx(); - addrModeCostSz += base->GetCostSz(); - addrmodeCount++; - } - - if (idx) - { - addrModeCostEx += idx->GetCostEx(); - addrModeCostSz += idx->GetCostSz(); - addrmodeCount++; - } - - if (cns) - { - if (((signed char)cns) == ((int)cns)) - { - addrModeCostSz += 1; - } - else - { - addrModeCostSz += 4; - } - addrmodeCount++; - } - if (mul) - { - addrmodeCount++; - } - // When we form a complex addressing mode we can reduced the costs - // associated with the interior GT_ADD and GT_LSH nodes: - // - // GT_ADD -- reduce this interior GT_ADD by (-3,-3) - // / \ -- - // GT_ADD 'cns' -- reduce this interior GT_ADD by (-2,-2) - // / \ -- - // 'base' GT_LSL -- reduce this interior GT_LSL by (-1,-1) - // / \ -- - // 'idx' 'mul' - // - if (addrmodeCount > 1) - { - // The number of interior GT_ADD and GT_LSL will always be one less than addrmodeCount - // - addrmodeCount--; - - GenTree* tmp = addr; - while (addrmodeCount > 0) - { - // decrement the gtCosts for the interior GT_ADD or GT_LSH node by the remaining - // addrmodeCount - tmp->SetCosts(tmp->GetCostEx() - addrmodeCount, tmp->GetCostSz() - addrmodeCount); - - addrmodeCount--; - if (addrmodeCount > 0) - { - GenTree* tmpOp1 = tmp->AsOp()->gtOp1; - GenTree* tmpOp2 = tmp->gtGetOp2(); - assert(tmpOp2 != nullptr); - - if ((tmpOp1 != base) && (tmpOp1->OperGet() == GT_ADD)) - { - tmp = tmpOp1; - } - else if (tmpOp2->OperGet() == GT_LSH) - { - tmp = tmpOp2; - } - else if (tmpOp1->OperGet() == GT_LSH) - { - tmp = tmpOp1; - } - else if (tmpOp2->OperGet() == GT_ADD) - { - tmp = tmpOp2; - } - else - { - // We can very rarely encounter a tree that has a GT_COMMA node - // that is difficult to walk, so we just early out without decrementing. - addrmodeCount = 0; - } - } - } - } -#elif defined TARGET_ARM - if (base) - { - addrModeCostEx += base->GetCostEx(); - addrModeCostSz += base->GetCostSz(); - if ((base->gtOper == GT_LCL_VAR) && ((idx == NULL) || (cns == 0))) - { - addrModeCostSz -= 1; - } - } - - if (idx) - { - addrModeCostEx += idx->GetCostEx(); - addrModeCostSz += idx->GetCostSz(); - if (mul > 0) - { - addrModeCostSz += 2; - } - } - - if (cns) - { - if (cns >= 128) // small offsets fits into a 16-bit instruction - { - if (cns < 4096) // medium offsets require a 32-bit instruction - { - if (!varTypeIsFloating(type)) - { - addrModeCostSz += 2; - } - } - else - { - addrModeCostEx += 2; // Very large offsets require movw/movt instructions - addrModeCostSz += 8; - } - } - } -#elif defined TARGET_ARM64 - if (base) - { - addrModeCostEx += base->GetCostEx(); - addrModeCostSz += base->GetCostSz(); - } - - if (idx) - { - addrModeCostEx += idx->GetCostEx(); - addrModeCostSz += idx->GetCostSz(); - } - - if (cns != 0) - { - if (cns >= (4096 * genTypeSize(type))) - { - addrModeCostEx += 1; - addrModeCostSz += 4; - } - } -#elif defined(TARGET_LOONGARCH64) - if (base) - { - addrModeCostEx += base->GetCostEx(); - addrModeCostSz += base->GetCostSz(); - } - - if (idx) - { - addrModeCostEx += idx->GetCostEx(); - addrModeCostSz += idx->GetCostSz(); - } - if (cns != 0) - { - if (!emitter::isValidSimm12(cns)) - { - // TODO-LoongArch64-CQ: tune for LoongArch64. - addrModeCostEx += 1; - addrModeCostSz += 4; - } - } -#elif defined(TARGET_RISCV64) - if (base) - { - addrModeCostEx += base->GetCostEx(); - addrModeCostSz += base->GetCostSz(); - } - - if (idx) - { - addrModeCostEx += idx->GetCostEx(); - addrModeCostSz += idx->GetCostSz(); - } - if (cns != 0) - { - if (!emitter::isValidSimm12(cns)) - { - // TODO-RISCV64-CQ: tune for RISCV64. - addrModeCostEx += 1; - addrModeCostSz += 4; - } - } -#else -#error "Unknown TARGET" -#endif - - assert(addr->gtOper == GT_ADD); - assert(!addr->gtOverflow()); - assert(mul != 1); - - // If we have an addressing mode, we have one of: - // [base + cns] - // [ idx * mul ] // mul >= 2, else we would use base instead of idx - // [ idx * mul + cns] // mul >= 2, else we would use base instead of idx - // [base + idx * mul ] // mul can be 0, 2, 4, or 8 - // [base + idx * mul + cns] // mul can be 0, 2, 4, or 8 - // Note that mul == 0 is semantically equivalent to mul == 1. - // Note that cns can be zero. - - assert((base != nullptr) || (idx != nullptr && mul >= 2)); - - // Walk 'addr' identifying non-overflow ADDs that will be part of the address mode. - // Note that we will be modifying 'op1' and 'op2' so that eventually they should - // map to the base and index. - GenTree* op1 = addr; - GenTree* op2 = nullptr; - gtWalkOp(&op1, &op2, base, false); - - // op1 and op2 are now descendents of the root GT_ADD of the addressing mode. -#if defined(TARGET_XARCH) - // Walk the operands again (the third operand is unused in this case). - // This time we will only consider adds with constant op2's, since - // we have already found either a non-ADD op1 or a non-constant op2. - // NOTE: we don't support ADD(op1, cns) addressing for ARM/ARM64 yet so - // this walk makes no sense there. - gtWalkOp(&op1, &op2, nullptr, true); - - // For XARCH we will fold GT_ADDs in the op2 position into the addressing mode, so we call - // gtWalkOp on both operands of the original GT_ADD. - // This is not done for ARMARCH. Though the stated reason is that we don't try to create a - // scaled index, in fact we actually do create them (even base + index*scale + offset). - - // At this point, 'op2' may itself be an ADD of a constant that should be folded - // into the addressing mode. - // Walk op2 looking for non-overflow GT_ADDs of constants. - gtWalkOp(&op2, &op1, nullptr, true); -#endif // defined(TARGET_XARCH) - - if ((mul > 1) && (op2 != nullptr) && op2->OperIs(GT_LSH, GT_MUL)) - { - op2->gtFlags |= GTF_ADDRMODE_NO_CSE; - } - - // Finally, adjust the costs on the parenting COMMAs. - while (addrComma != addr) - { - int addrCostExDelta = originalAddrCostEx - addrModeCostEx; - int addrCostSzDelta = originalAddrCostSz - addrModeCostSz; - addrComma->SetCosts(addrComma->GetCostEx() - addrCostExDelta, addrComma->GetCostSz() - addrCostSzDelta); - - *pCostEx += addrComma->AsOp()->gtGetOp1()->GetCostEx(); - *pCostSz += addrComma->AsOp()->gtGetOp1()->GetCostSz(); - - addrComma = addrComma->AsOp()->gtGetOp2(); - } - - *pCostEx += addrModeCostEx; - *pCostSz += addrModeCostSz; - - return true; - - } // if (genCreateAddrMode(...)) - - return false; -} - -static void SetIndirectStoreEvalOrder(Compiler* comp, GenTreeIndir* store, bool* allowReversal) -{ - assert(store->OperIs(GT_STORE_BLK, GT_STOREIND)); - - GenTree* addr = store->Addr(); - GenTree* data = store->Data(); - *allowReversal = true; - - if (addr->IsInvariant()) - { - *allowReversal = false; - store->SetReverseOp(); - return; - } - - if ((addr->gtFlags & GTF_ALL_EFFECT) != 0) - { - return; - } - - // In case op2 assigns to a local var that is used in op1, we have to evaluate op1 first. - if (comp->gtMayHaveStoreInterference(data, addr)) - { - // TODO-ASG-Cleanup: move this guard to "gtCanSwapOrder". - *allowReversal = false; - return; - } - - // If op2 is simple then evaluate op1 first - if (data->OperIsLeaf()) - { - return; - } - - *allowReversal = false; - store->SetReverseOp(); -} - -/***************************************************************************** - * - * Given a tree, figure out the order in which its sub-operands should be - * evaluated. If the second operand of a binary operator is more expensive - * than the first operand, then try to swap the operand trees. Updates the - * GTF_REVERSE_OPS bit if necessary in this case. - * - * Returns the Sethi 'complexity' estimate for this tree (the higher - * the number, the higher is the tree's resources requirement). - * - * This function sets: - * 1. GetCostEx() to the execution complexity estimate - * 2. GetCostSz() to the code size estimate - * 3. Sometimes sets GTF_ADDRMODE_NO_CSE on nodes in the tree. - */ - -#ifdef _PREFAST_ -#pragma warning(push) -#pragma warning(disable : 21000) // Suppress PREFast warning about overly large function -#endif -unsigned Compiler::gtSetEvalOrder(GenTree* tree) -{ - assert(tree); - - if (opts.OptimizationDisabled()) - { - return gtSetEvalOrderMinOpts(tree); - } - - /* Is this a FP value? */ - - bool isflt = varTypeIsFloating(tree->TypeGet()); - - /* Figure out what kind of a node we have */ - - const genTreeOps oper = tree->OperGet(); - const unsigned kind = tree->OperKind(); - - /* Assume no fixed registers will be trashed */ - - unsigned level; - int costEx; - int costSz; - -#ifdef DEBUG - costEx = -1; - costSz = -1; -#endif - - /* Is this a leaf node? */ - - if (kind & GTK_LEAF) - { - switch (oper) - { -#ifdef TARGET_ARM - case GT_CNS_STR: - // Uses movw/movt - costSz = 8; - costEx = 2; - goto COMMON_CNS; - - case GT_CNS_LNG: - { - GenTreeIntConCommon* con = tree->AsIntConCommon(); - - INT64 lngVal = con->LngValue(); - INT32 loVal = (INT32)(lngVal & 0xffffffff); - INT32 hiVal = (INT32)(lngVal >> 32); - - if (lngVal == 0) - { - costSz = 1; - costEx = 1; - } - else - { - // Minimum of one instruction to setup hiVal, - // and one instruction to setup loVal - costSz = 4 + 4; - costEx = 1 + 1; - - if (!codeGen->validImmForInstr(INS_mov, (target_ssize_t)hiVal) && - !codeGen->validImmForInstr(INS_mvn, (target_ssize_t)hiVal)) - { - // Needs extra instruction: movw/movt - costSz += 4; - costEx += 1; - } - - if (!codeGen->validImmForInstr(INS_mov, (target_ssize_t)loVal) && - !codeGen->validImmForInstr(INS_mvn, (target_ssize_t)loVal)) - { - // Needs extra instruction: movw/movt - costSz += 4; - costEx += 1; - } - } - goto COMMON_CNS; - } - - case GT_CNS_INT: - { - // If the constant is a handle then it will need to have a relocation - // applied to it. - // Any constant that requires a reloc must use the movw/movt sequence - // - GenTreeIntConCommon* con = tree->AsIntConCommon(); - target_ssize_t conVal = (target_ssize_t)con->IconValue(); - - if (con->ImmedValNeedsReloc(this)) - { - // Requires movw/movt - costSz = 8; - costEx = 2; - } - else if (codeGen->validImmForInstr(INS_add, conVal)) - { - // Typically included with parent oper - costSz = 2; - costEx = 1; - } - else if (codeGen->validImmForInstr(INS_mov, conVal) || codeGen->validImmForInstr(INS_mvn, conVal)) - { - // Uses mov or mvn - costSz = 4; - costEx = 1; - } - else - { - // Needs movw/movt - costSz = 8; - costEx = 2; - } - goto COMMON_CNS; - } - -#elif defined TARGET_XARCH - - case GT_CNS_STR: -#ifdef TARGET_AMD64 - costSz = 10; - costEx = 2; -#else // TARGET_X86 - costSz = 4; - costEx = 1; -#endif - goto COMMON_CNS; - - case GT_CNS_LNG: - case GT_CNS_INT: - { - GenTreeIntConCommon* con = tree->AsIntConCommon(); - ssize_t conVal = (oper == GT_CNS_LNG) ? (ssize_t)con->LngValue() : con->IconValue(); - bool fitsInVal = true; - -#ifdef TARGET_X86 - if (oper == GT_CNS_LNG) - { - INT64 lngVal = con->LngValue(); - - conVal = (ssize_t)lngVal; // truncate to 32-bits - - fitsInVal = ((INT64)conVal == lngVal); - } -#endif // TARGET_X86 - - // If the constant is a handle then it will need to have a relocation - // applied to it. - // - bool iconNeedsReloc = con->ImmedValNeedsReloc(this); - - if (iconNeedsReloc) - { - costSz = 4; - costEx = 1; - } - else if (fitsInVal && GenTreeIntConCommon::FitsInI8(conVal)) - { - costSz = 1; - costEx = 1; - } -#ifdef TARGET_AMD64 - else if (!GenTreeIntConCommon::FitsInI32(conVal)) - { - costSz = 10; - costEx = 2; - if (con->IsIconHandle()) - { - // A sort of a hint for CSE to try harder for class handles - costEx += 1; - } - } -#endif // TARGET_AMD64 - else - { - costSz = 4; - costEx = 1; - } -#ifdef TARGET_X86 - if (oper == GT_CNS_LNG) - { - costSz += fitsInVal ? 1 : 4; - costEx += 1; - } -#endif // TARGET_X86 - - goto COMMON_CNS; - } - -#elif defined(TARGET_ARM64) - - case GT_CNS_STR: - case GT_CNS_LNG: - case GT_CNS_INT: - { - GenTreeIntConCommon* con = tree->AsIntConCommon(); - bool iconNeedsReloc = con->ImmedValNeedsReloc(this); - INT64 imm = con->LngValue(); - emitAttr size = EA_SIZE(emitActualTypeSize(tree)); - - if (iconNeedsReloc) - { - costSz = 8; - costEx = 2; - } - else if (emitter::emitIns_valid_imm_for_add(imm, size)) - { - costSz = 2; - costEx = 1; - } - else if (emitter::emitIns_valid_imm_for_mov(imm, size)) - { - costSz = 4; - costEx = 1; - } - else - { - // Arm64 allows any arbitrary 16-bit constant to be loaded into a register halfword - // There are three forms - // movk which loads into any halfword preserving the remaining halfwords - // movz which loads into any halfword zeroing the remaining halfwords - // movn which loads into any halfword zeroing the remaining halfwords then bitwise inverting - // the register - // In some cases it is preferable to use movn, because it has the side effect of filling the - // other halfwords - // with ones - - // Determine whether movn or movz will require the fewest instructions to populate the immediate - bool preferMovz = false; - bool preferMovn = false; - int instructionCount = 4; - - for (int i = (size == EA_8BYTE) ? 48 : 16; i >= 0; i -= 16) - { - if (!preferMovn && (uint16_t(imm >> i) == 0x0000)) - { - preferMovz = true; // by using a movk to start we can save one instruction - instructionCount--; - } - else if (!preferMovz && (uint16_t(imm >> i) == 0xffff)) - { - preferMovn = true; // by using a movn to start we can save one instruction - instructionCount--; - } - } - - costEx = instructionCount; - costSz = 4 * instructionCount; - } - } - goto COMMON_CNS; - -#elif defined(TARGET_LOONGARCH64) - // TODO-LoongArch64-CQ: tune the costs. - case GT_CNS_STR: - costEx = IND_COST_EX + 2; - costSz = 4; - goto COMMON_CNS; - - case GT_CNS_LNG: - case GT_CNS_INT: - costEx = 1; - costSz = 4; - goto COMMON_CNS; -#elif defined(TARGET_RISCV64) - // TODO-RISCV64-CQ: tune the costs. - case GT_CNS_STR: - costEx = IND_COST_EX + 2; - costSz = 4; - goto COMMON_CNS; - - case GT_CNS_LNG: - case GT_CNS_INT: - costEx = 1; - costSz = 4; - goto COMMON_CNS; -#else - case GT_CNS_STR: - case GT_CNS_LNG: - case GT_CNS_INT: -#error "Unknown TARGET" -#endif - COMMON_CNS: - /* - Note that some code below depends on constants always getting - moved to be the second operand of a binary operator. This is - easily accomplished by giving constants a level of 0, which - we do on the next line. If you ever decide to change this, be - aware that unless you make other arrangements for integer - constants to be moved, stuff will break. - */ - - level = 0; - break; - - case GT_CNS_DBL: - { - level = 0; -#if defined(TARGET_XARCH) - if (tree->IsFloatPositiveZero() || tree->IsFloatAllBitsSet()) - { - // We generate `xorp* tgtReg, tgtReg` for PositiveZero and - // `pcmpeqd tgtReg, tgtReg` for AllBitsSet which is 3-5 bytes - // but which can be elided by the instruction decoder. - - costEx = 1; - costSz = 2; - } - else - { - // We generate `movs* tgtReg, [mem]` which is 4-6 bytes - // and which has the same cost as an indirection. - - costEx = IND_COST_EX; - costSz = 2; - } -#elif defined(TARGET_ARM) - var_types targetType = tree->TypeGet(); - if (targetType == TYP_FLOAT) - { - costEx = 1 + 2; - costSz = 2 + 4; - } - else - { - assert(targetType == TYP_DOUBLE); - costEx = 1 + 4; - costSz = 2 + 8; - } -#elif defined(TARGET_ARM64) - if (tree->IsFloatPositiveZero() || emitter::emitIns_valid_imm_for_fmov(tree->AsDblCon()->DconValue())) - { - // Zero and certain other immediates can be specially created with a single instruction - // These can be cheaply reconstituted but still take up 4-bytes of native codegen - - costEx = 1; - costSz = 2; - } - else - { - // We load the constant from memory and so will take the same cost as GT_IND - - costEx = IND_COST_EX; - costSz = 2; - } -#elif defined(TARGET_LOONGARCH64) - // TODO-LoongArch64-CQ: tune the costs. - costEx = 2; - costSz = 8; -#elif defined(TARGET_RISCV64) - // TODO-RISCV64-CQ: tune the costs. - costEx = 2; - costSz = 8; -#else -#error "Unknown TARGET" -#endif - } - break; - -#if defined(FEATURE_SIMD) - case GT_CNS_VEC: - { - level = 0; - - if (tree->AsVecCon()->IsAllBitsSet() || tree->AsVecCon()->IsZero()) - { - // We generate `cmpeq* tgtReg, tgtReg`, which is 4-5 bytes, for AllBitsSet - // and generate `xorp* tgtReg, tgtReg`, which is 3-5 bytes, for Zero - // both of which can be elided by the instruction decoder. - - costEx = 1; - costSz = 2; - } - else - { - // We generate `movup* tgtReg, [mem]` which is 4-6 bytes - // and which has the same cost as an indirection. - - costEx = IND_COST_EX; - costSz = 2; - } - break; - } -#endif // FEATURE_SIMD - -#if defined(FEATURE_MASKED_HW_INTRINSICS) - case GT_CNS_MSK: - { - level = 0; - - if (tree->AsMskCon()->IsAllBitsSet() || tree->AsMskCon()->IsZero()) - { - costEx = 1; - costSz = 2; - } - else - { - costEx = IND_COST_EX; - costSz = 2; - } - break; - } -#endif // FEATURE_MASKED_HW_INTRINSICS - - case GT_LCL_VAR: - level = 1; - gtGetLclVarNodeCost(tree->AsLclVar(), &costEx, &costSz, gtIsLikelyRegVar(tree)); - break; - - case GT_LCL_FLD: - level = 1; - gtGetLclFldNodeCost(tree->AsLclFld(), &costEx, &costSz); - break; - - case GT_LCL_ADDR: - level = 1; - costEx = 3; - costSz = 3; - break; - - case GT_PHI_ARG: - level = 0; - costEx = 0; - costSz = 0; - break; - - case GT_NOP: - level = 0; - costEx = 0; - costSz = 0; - break; - - default: - level = 1; - costEx = 1; - costSz = 1; - break; - } - goto DONE; - } - - /* Is it a 'simple' unary/binary operator? */ - - if (kind & GTK_SMPOP) - { - GenTree* op1 = tree->AsOp()->gtOp1; - GenTree* op2 = tree->gtGetOp2IfPresent(); - - costEx = 0; - costSz = 0; - - if (tree->OperIsAddrMode()) - { - if (op1 == nullptr) - { - op1 = op2; - op2 = nullptr; - } - } - - /* Check for a nilary operator */ - - if (op1 == nullptr) - { - assert(op2 == nullptr); - - level = 0; - - goto DONE; - } - - /* Is this a unary operator? */ - - if (op2 == nullptr) - { - /* Process the operand of the operator */ - - /* Most Unary ops have costEx of 1 */ - costEx = 1; - costSz = 1; - - level = gtSetEvalOrder(op1); - - GenTreeIntrinsic* intrinsic; - - /* Special handling for some operators */ - - switch (oper) - { - case GT_JTRUE: - costEx = 2; - costSz = 2; - break; - - case GT_SWITCH: - costEx = 10; - costSz = 5; - break; - - case GT_CAST: -#if defined(TARGET_ARM) - costEx = 1; - costSz = 1; - if (isflt || varTypeIsFloating(op1->TypeGet())) - { - costEx = 3; - costSz = 4; - } -#elif defined(TARGET_ARM64) - costEx = 1; - costSz = 2; - if (isflt || varTypeIsFloating(op1->TypeGet())) - { - costEx = 2; - costSz = 4; - } -#elif defined(TARGET_XARCH) - costEx = 1; - costSz = 2; - - if (isflt || varTypeIsFloating(op1->TypeGet())) - { - /* cast involving floats always go through memory */ - costEx = IND_COST_EX * 2; - costSz = 6; - } -#elif defined(TARGET_LOONGARCH64) - // TODO-LoongArch64-CQ: tune the costs. - costEx = 1; - costSz = 4; -#elif defined(TARGET_RISCV64) - // TODO-RISCV64-CQ: tune the costs. - costEx = 1; - costSz = 4; -#else -#error "Unknown TARGET" -#endif - - /* Overflow casts are a lot more expensive */ - if (tree->gtOverflow()) - { - costEx += 6; - costSz += 6; - } - - break; - - case GT_INTRINSIC: - intrinsic = tree->AsIntrinsic(); - // named intrinsic - assert(intrinsic->gtIntrinsicName != NI_Illegal); - - // GT_INTRINSIC intrinsics Sin, Cos, Sqrt, Abs ... have higher costs. - // TODO: tune these costs target specific as some of these are - // target intrinsics and would cost less to generate code. - switch (intrinsic->gtIntrinsicName) - { - default: - assert(!"missing case for gtIntrinsicName"); - costEx = 12; - costSz = 12; - break; - - case NI_System_Math_Abs: - costEx = 5; - costSz = 15; - break; - - case NI_System_Math_Acos: - case NI_System_Math_Acosh: - case NI_System_Math_Asin: - case NI_System_Math_Asinh: - case NI_System_Math_Atan: - case NI_System_Math_Atanh: - case NI_System_Math_Atan2: - case NI_System_Math_Cbrt: - case NI_System_Math_Ceiling: - case NI_System_Math_Cos: - case NI_System_Math_Cosh: - case NI_System_Math_Exp: - case NI_System_Math_Floor: - case NI_System_Math_FusedMultiplyAdd: - case NI_System_Math_ILogB: - case NI_System_Math_Log: - case NI_System_Math_Log2: - case NI_System_Math_Log10: - case NI_System_Math_Max: - case NI_System_Math_MaxMagnitude: - case NI_System_Math_MaxMagnitudeNumber: - case NI_System_Math_MaxNumber: - case NI_System_Math_Min: - case NI_System_Math_MinMagnitude: - case NI_System_Math_MinMagnitudeNumber: - case NI_System_Math_MinNumber: - case NI_System_Math_Pow: - case NI_System_Math_Round: - case NI_System_Math_Sin: - case NI_System_Math_Sinh: - case NI_System_Math_Sqrt: - case NI_System_Math_Tan: - case NI_System_Math_Tanh: - case NI_System_Math_Truncate: - { - // Giving intrinsics a large fixed execution cost is because we'd like to CSE - // them, even if they are implemented by calls. This is different from modeling - // user calls since we never CSE user calls. We don't do this for target intrinsics - // however as they typically represent single instruction calls - - if (IsIntrinsicImplementedByUserCall(intrinsic->gtIntrinsicName)) - { - costEx = 36; - costSz = 4; - } - else - { - costEx = 3; - costSz = 4; - } - break; - } - - case NI_System_Object_GetType: - // Giving intrinsics a large fixed execution cost is because we'd like to CSE - // them, even if they are implemented by calls. This is different from modeling - // user calls since we never CSE user calls. - costEx = 36; - costSz = 4; - break; - -#if defined(FEATURE_SIMD) - case NI_SIMD_UpperRestore: - case NI_SIMD_UpperSave: - { - // TODO-CQ: 1 Ex/Sz isn't necessarily "accurate" but it is what the previous - // cost was computed as, in gtSetMultiOpOrder, when this was handled by the - // older SIMD intrinsic support. - - costEx = 1; - costSz = 1; - break; - } -#endif // FEATURE_SIMD - } - level++; - break; - - case GT_NOT: - case GT_NEG: - // We need to ensure that -x is evaluated before x or else - // we get burned while adjusting genFPstkLevel in x*-x where - // the rhs x is the last use of the enregistered x. - // - // Even in the integer case we want to prefer to - // evaluate the side without the GT_NEG node, all other things - // being equal. Also a GT_NOT requires a scratch register - - level++; - break; - - case GT_ARR_LENGTH: - case GT_MDARR_LENGTH: - case GT_MDARR_LOWER_BOUND: - level++; - - // Array meta-data access should be the same as an indirection, which has a costEx of IND_COST_EX. - costEx = IND_COST_EX - 1; - costSz = 2; - break; - - case GT_BLK: - // We estimate the cost of a GT_BLK to be two loads (GT_INDs) - costEx = 2 * IND_COST_EX; - costSz = 2 * 2; - break; - - case GT_BOX: - // We estimate the cost of a GT_BOX to be two stores (GT_INDs) - costEx = 2 * IND_COST_EX; - costSz = 2 * 2; - break; - - case GT_ARR_ADDR: - costEx = 0; - costSz = 0; - - // To preserve previous behavior, we will always use "gtMarkAddrMode" for ARR_ADDR. - if (op1->OperIs(GT_ADD) && gtMarkAddrMode(op1, &costEx, &costSz, tree->AsArrAddr()->GetElemType())) - { - op1->SetCosts(costEx, costSz); - goto DONE; - } - break; - - case GT_IND: - // An indirection should always have a non-zero level. - // Only constant leaf nodes have level 0. - if (level == 0) - { - level = 1; - } - - if (gtGetIndNodeCost(tree->AsIndir(), &costEx, &costSz)) - { - goto DONE; - } - break; - - case GT_STORE_LCL_VAR: - if (gtIsLikelyRegVar(tree)) - { - // Store to an enregistered local. - costEx = op1->GetCostEx(); - costSz = max(3, (int)op1->GetCostSz()); // 3 is an estimate for a reg-reg move. - goto DONE; - } - - gtGetLclVarNodeCost(tree->AsLclVar(), &costEx, &costSz, /* isLikelyRegVar */ false); - goto SET_LCL_STORE_COSTS; - - case GT_STORE_LCL_FLD: - gtGetLclFldNodeCost(tree->AsLclFld(), &costEx, &costSz); - - SET_LCL_STORE_COSTS: - costEx += 1; - costSz += 1; -#ifdef TARGET_ARM - if (isflt) - { - costSz += 2; - } -#endif // TARGET_ARM -#ifndef TARGET_64BIT - if (tree->TypeIs(TYP_LONG)) - { - // Operations on longs are more expensive. - costEx += 3; - costSz += 3; - } -#endif // !TARGET_64BIT - break; - - default: - break; - } - costEx += op1->GetCostEx(); - costSz += op1->GetCostSz(); - goto DONE; - } - - /* Binary operator - check for certain special cases */ - bool includeOp1Cost = true; - bool includeOp2Cost = true; - bool allowReversal = true; - - /* Default Binary ops have a cost of 1,1 */ - costEx = 1; - costSz = 1; - -#ifdef TARGET_ARM - if (isflt) - { - costSz += 2; - } -#endif -#ifndef TARGET_64BIT - if (varTypeIsLong(op1->TypeGet())) - { - /* Operations on longs are more expensive */ - costEx += 3; - costSz += 3; - } -#endif - level = gtSetEvalOrder(op1); - unsigned lvl2 = gtSetEvalOrder(op2); - - switch (oper) - { - case GT_MOD: - case GT_UMOD: - - /* Modulo by a power of 2 is easy */ - - if (op2->IsCnsIntOrI()) - { - size_t ival = op2->AsIntConCommon()->IconValue(); - - if (ival > 0 && ival == genFindLowestBit(ival)) - { - break; - } - } - - FALLTHROUGH; - - case GT_DIV: - case GT_UDIV: - - if (isflt) - { - /* fp division is very expensive to execute */ - costEx = 36; // TYP_DOUBLE - costSz += 3; - } - else - { - /* integer division is also very expensive */ - costEx = 20; - costSz += 2; - - // Encourage the first operand to be evaluated (into EAX/EDX) first */ - level += 3; - } - break; - - case GT_MUL: - - if (isflt) - { - /* FP multiplication instructions are more expensive */ - costEx += 4; - costSz += 3; - } - else - { - /* Integer multiplication instructions are more expensive */ - costEx += 3; - costSz += 2; - - if (tree->gtOverflow()) - { - /* Overflow check are more expensive */ - costEx += 3; - costSz += 3; - } - -#ifdef TARGET_X86 - if ((tree->gtType == TYP_LONG) || tree->gtOverflow()) - { - /* We use imulEAX for TYP_LONG and overflow multiplications */ - // Encourage the first operand to be evaluated (into EAX/EDX) first */ - level += 4; - - /* The 64-bit imul instruction costs more */ - costEx += 4; - } -#endif // TARGET_X86 - } - break; - - case GT_ADD: - case GT_SUB: - if (isflt) - { - /* FP instructions are a bit more expensive */ - costEx += 4; - costSz += 3; - break; - } - - /* Overflow check are more expensive */ - if (tree->gtOverflow()) - { - costEx += 3; - costSz += 3; - } - break; - - case GT_LSH: - case GT_RSH: - case GT_RSZ: - case GT_ROL: - case GT_ROR: - /* Variable sized shifts are more expensive and use REG_SHIFT */ - - if (!op2->IsCnsIntOrI()) - { - costEx += 3; -#ifndef TARGET_64BIT - // Variable sized LONG shifts require the use of a helper call - // - if (tree->gtType == TYP_LONG) - { - level += 5; - lvl2 += 5; - costEx += 3 * IND_COST_EX; - costSz += 4; - } -#endif // !TARGET_64BIT - } - break; - - case GT_EQ: - case GT_NE: - case GT_LT: - case GT_LE: - case GT_GE: - case GT_GT: - /* Float compares remove both operands from the FP stack */ - /* Also FP comparison uses EAX for flags */ - - if (varTypeIsFloating(op1->TypeGet())) - { - level++; - lvl2++; - } - if ((tree->gtFlags & GTF_RELOP_JMP_USED) == 0) - { - /* Using a setcc instruction is more expensive */ - costEx += 3; - } - break; - - case GT_BOUNDS_CHECK: - costEx = 4; // cmp reg,reg and jae throw (not taken) - costSz = 7; // jump to cold section - // Bounds check nodes used to not be binary, thus GTF_REVERSE_OPS was not enabled for them. This - // condition preserves that behavior. Additionally, CQ analysis shows that enabling GTF_REVERSE_OPS - // for these nodes leads to mixed results at best. - allowReversal = false; - break; - - case GT_INTRINSIC: - // We do not swap operand execution order for intrinsics that are implemented by user calls because - // of trickiness around ensuring the execution order does not change during rationalization. - if (IsIntrinsicImplementedByUserCall(tree->AsIntrinsic()->gtIntrinsicName)) - { - allowReversal = false; - } - - switch (tree->AsIntrinsic()->gtIntrinsicName) - { - case NI_System_Math_Atan2: - case NI_System_Math_Pow: - // These math intrinsics are actually implemented by user calls. Increase the - // Sethi 'complexity' by two to reflect the argument register requirement. - level += 2; - break; - - case NI_System_Math_Max: - case NI_System_Math_MaxMagnitude: - case NI_System_Math_MaxMagnitudeNumber: - case NI_System_Math_MaxNumber: - case NI_System_Math_Min: - case NI_System_Math_MinMagnitude: - case NI_System_Math_MinMagnitudeNumber: - case NI_System_Math_MinNumber: - { - level++; - break; - } - - default: - assert(!"Unknown binary GT_INTRINSIC operator"); - break; - } - break; - - case GT_COMMA: - /* Comma tosses the result of the left operand */ - level = lvl2; - /* GT_COMMA cost is the sum of op1 and op2 costs */ - costEx = (op1->GetCostEx() + op2->GetCostEx()); - costSz = (op1->GetCostSz() + op2->GetCostSz()); - goto DONE; - - case GT_INDEX_ADDR: - costEx = 6; // cmp reg,reg; jae throw; mov reg, [addrmode] (not taken) - costSz = 9; // jump to cold section - break; - - case GT_STORE_BLK: - // We estimate the cost of a GT_STORE_BLK to be two stores. - costEx += 2 * IND_COST_EX; - costSz += 2 * 2; - goto SET_INDIRECT_STORE_ORDER; - - case GT_STOREIND: - if (gtGetIndNodeCost(tree->AsIndir(), &costEx, &costSz)) - { - includeOp1Cost = false; - } - - costEx += 1; - costSz += 1; -#ifndef TARGET_64BIT - if (tree->TypeIs(TYP_LONG)) - { - // Operations on longs are more expensive. - costEx += 3; - costSz += 3; - } -#endif // !TARGET_64BIT - - SET_INDIRECT_STORE_ORDER: - // TODO-ASG-Cleanup: this logic emulated the ASG case below. See how of much of it can be deleted. - if (!optValnumCSE_phase || optCSE_canSwap(op1, op2)) - { - SetIndirectStoreEvalOrder(this, tree->AsIndir(), &allowReversal); - } - break; - - default: - break; - } - - if (includeOp1Cost) - { - costEx += op1->GetCostEx(); - costSz += op1->GetCostSz(); - } - if (includeOp2Cost) - { - costEx += op2->GetCostEx(); - costSz += op2->GetCostSz(); - } - - /* We need to evaluate constants later as many places in codegen - can't handle op1 being a constant. This is normally naturally - enforced as constants have the least level of 0. However, - sometimes we end up with a tree like "cns1 < nop(cns2)". In - such cases, both sides have a level of 0. So encourage constants - to be evaluated last in such cases */ - - if ((level == 0) && (level == lvl2) && op1->OperIsConst() && - (tree->OperIsCommutative() || tree->OperIsCompare())) - { - lvl2++; - } - - // We try to swap operands if the second one is more expensive. - // Don't swap anything if we're in linear order; we're really just interested in the costs. - bool tryToSwap = false; - if (allowReversal && (fgOrder != FGOrderLinear)) - { - if (tree->IsReverseOp()) - { - tryToSwap = (level > lvl2); - } - else - { - tryToSwap = (level < lvl2); - } - - // Try to force extra swapping when in the stress mode: - if (compStressCompile(STRESS_REVERSE_FLAG, 60) && !tree->IsReverseOp() && !op2->OperIsConst()) - { - tryToSwap = true; - } - } - - if (tryToSwap) - { - bool canSwap = tree->IsReverseOp() ? gtCanSwapOrder(op2, op1) : gtCanSwapOrder(op1, op2); - - if (canSwap) - { - /* Can we swap the order by commuting the operands? */ - - switch (oper) - { - case GT_EQ: - case GT_NE: - case GT_LT: - case GT_LE: - case GT_GE: - case GT_GT: - if (GenTree::SwapRelop(oper) != oper) - { - tree->SetOper(GenTree::SwapRelop(oper), GenTree::PRESERVE_VN); - } - - FALLTHROUGH; - - case GT_ADD: - case GT_MUL: - - case GT_OR: - case GT_XOR: - case GT_AND: - - /* Swap the operands */ - - tree->AsOp()->gtOp1 = op2; - tree->AsOp()->gtOp2 = op1; - break; - - case GT_QMARK: - case GT_COLON: - break; - - default: - // Mark the operand's evaluation order to be swapped. - tree->gtFlags ^= GTF_REVERSE_OPS; - break; - } - } - } - - /* Swap the level counts */ - if (tree->IsReverseOp()) - { - std::swap(level, lvl2); - } - - /* Compute the sethi number for this binary operator */ - - if (level < 1) - { - level = lvl2; - } - else if (level == lvl2) - { - level += 1; - } - - goto DONE; - } - - /* See what kind of a special operator we have here */ - - switch (oper) - { - unsigned lvl2; // Scratch variable - - case GT_CALL: - - assert(tree->gtFlags & GTF_CALL); - - level = 0; - costEx = 5; - costSz = 2; - - GenTreeCall* call; - call = tree->AsCall(); - - // Evaluate the arguments - - lvl2 = gtSetCallArgsOrder(&call->gtArgs, /* lateArgs */ false, &costEx, &costSz); - if (level < lvl2) - { - level = lvl2; - } - - // Evaluate the temp register arguments list - - lvl2 = gtSetCallArgsOrder(&call->gtArgs, /* lateArgs */ true, &costEx, &costSz); - if (level < lvl2) - { - level = lvl2; - } - - if (call->gtCallType == CT_INDIRECT) - { - // pinvoke-calli cookie is a constant, or constant indirection - assert(call->gtCallCookie == nullptr || call->gtCallCookie->gtOper == GT_CNS_INT || - call->gtCallCookie->gtOper == GT_IND); - - GenTree* indirect = call->gtCallAddr; - - lvl2 = gtSetEvalOrder(indirect); - if (level < lvl2) - { - level = lvl2; - } - costEx += indirect->GetCostEx() + IND_COST_EX; - costSz += indirect->GetCostSz(); - } - else - { - if (call->IsVirtual()) - { - GenTree* controlExpr = call->gtControlExpr; - if (controlExpr != nullptr) - { - lvl2 = gtSetEvalOrder(controlExpr); - if (level < lvl2) - { - level = lvl2; - } - costEx += controlExpr->GetCostEx(); - costSz += controlExpr->GetCostSz(); - } - } -#ifdef TARGET_ARM - if (call->IsVirtualStub()) - { - // We generate movw/movt/ldr - costEx += (1 + IND_COST_EX); - costSz += 8; - if (call->gtCallMoreFlags & GTF_CALL_M_VIRTSTUB_REL_INDIRECT) - { - // Must use R12 for the ldr target -- REG_JUMP_THUNK_PARAM - costSz += 2; - } - } - else if (!opts.jitFlags->IsSet(JitFlags::JIT_FLAG_PREJIT)) - { - costEx += 2; - costSz += 6; - } - costSz += 2; -#endif - -#ifdef TARGET_XARCH - costSz += 3; -#endif - } - - level += 1; - - /* Virtual calls are a bit more expensive */ - if (call->IsVirtual()) - { - costEx += 2 * IND_COST_EX; - costSz += 2; - } - - level += 5; - costEx += 3 * IND_COST_EX; - break; - -#if defined(FEATURE_HW_INTRINSICS) - case GT_HWINTRINSIC: - return gtSetMultiOpOrder(tree->AsMultiOp()); -#endif // FEATURE_HW_INTRINSICS - - case GT_ARR_ELEM: - { - GenTreeArrElem* arrElem = tree->AsArrElem(); - - level = gtSetEvalOrder(arrElem->gtArrObj); - costEx = arrElem->gtArrObj->GetCostEx(); - costSz = arrElem->gtArrObj->GetCostSz(); - - for (unsigned dim = 0; dim < arrElem->gtArrRank; dim++) - { - lvl2 = gtSetEvalOrder(arrElem->gtArrInds[dim]); - if (level < lvl2) - { - level = lvl2; - } - costEx += arrElem->gtArrInds[dim]->GetCostEx(); - costSz += arrElem->gtArrInds[dim]->GetCostSz(); - } - - level += arrElem->gtArrRank; - costEx += 2 + (arrElem->gtArrRank * (IND_COST_EX + 1)); - costSz += 2 + (arrElem->gtArrRank * 2); - } - break; - - case GT_PHI: - for (GenTreePhi::Use& use : tree->AsPhi()->Uses()) - { - lvl2 = gtSetEvalOrder(use.GetNode()); - // PHI args should always have cost 0 and level 0 - assert(lvl2 == 0); - assert(use.GetNode()->GetCostEx() == 0); - assert(use.GetNode()->GetCostSz() == 0); - } - level = 1; - costEx = 0; - costSz = 0; - break; - - case GT_FIELD_LIST: - level = 0; - costEx = 0; - costSz = 0; - for (GenTreeFieldList::Use& use : tree->AsFieldList()->Uses()) - { - unsigned opLevel = gtSetEvalOrder(use.GetNode()); - level = max(level, opLevel); - gtSetEvalOrder(use.GetNode()); - costEx += use.GetNode()->GetCostEx(); - costSz += use.GetNode()->GetCostSz(); - } - break; - - case GT_CMPXCHG: - { - GenTree* addr = tree->AsCmpXchg()->Addr(); - level = gtSetEvalOrder(addr); - costSz = addr->GetCostSz(); - - GenTree* value = tree->AsCmpXchg()->Data(); - lvl2 = gtSetEvalOrder(value); - if (level < lvl2) - { - level = lvl2; - } - costSz += value->GetCostSz(); - - GenTree* comparand = tree->AsCmpXchg()->Comparand(); - lvl2 = gtSetEvalOrder(comparand); - if (level < lvl2) - { - level = lvl2; - } - costSz += comparand->GetCostSz(); - - costEx = MAX_COST; // Seriously, what could be more expensive than lock cmpxchg? - costSz += 5; // size of lock cmpxchg [reg+C], reg - } - break; - - case GT_SELECT: - level = gtSetEvalOrder(tree->AsConditional()->gtCond); - costEx = tree->AsConditional()->gtCond->GetCostEx(); - costSz = tree->AsConditional()->gtCond->GetCostSz(); - - lvl2 = gtSetEvalOrder(tree->AsConditional()->gtOp1); - level = max(level, lvl2); - costEx += tree->AsConditional()->gtOp1->GetCostEx(); - costSz += tree->AsConditional()->gtOp1->GetCostSz(); - - lvl2 = gtSetEvalOrder(tree->AsConditional()->gtOp2); - level = max(level, lvl2); - costEx += tree->AsConditional()->gtOp2->GetCostEx(); - costSz += tree->AsConditional()->gtOp2->GetCostSz(); - - costEx += 1; - costSz += 1; - break; - - default: - JITDUMP("unexpected operator in this tree:\n"); - DISPTREE(tree); - - NO_WAY("unexpected operator"); - } - -DONE: - // Some path through this function must have set the costs. - assert(costEx != -1); - assert(costSz != -1); - - tree->SetCosts(costEx, costSz); - - return level; -} -#ifdef _PREFAST_ -#pragma warning(pop) -#endif - -//------------------------------------------------------------------------ -// gtSetEvalOrderMinOpts: A MinOpts specific version of gtSetEvalOrder. We don't -// need to set costs, but we're looking for opportunities to swap operands. -// -// Arguments: -// tree - The tree for which we are setting the evaluation order. -// -// Return Value: -// the Sethi 'complexity' estimate for this tree (the higher -// the number, the higher is the tree's resources requirement) -// -unsigned Compiler::gtSetEvalOrderMinOpts(GenTree* tree) -{ - assert(tree); - if (fgOrder == FGOrderLinear) - { - // We don't re-order operands in LIR anyway. - return 0; - } - - if (tree->OperIsLeaf()) - { - // Nothing to do for leaves, report as having Sethi 'complexity' of 0 - return 0; - } - - unsigned level = 1; - if (tree->OperIsSimple()) - { - GenTree* op1 = tree->AsOp()->gtOp1; - GenTree* op2 = tree->gtGetOp2IfPresent(); - - // Only GT_LEA may have a nullptr op1 and a non-nullptr op2 - if (tree->OperIs(GT_LEA) && (op1 == nullptr)) - { - std::swap(op1, op2); - } - - // Check for a nilary operator - if (op1 == nullptr) - { - // E.g. void GT_RETURN, GT_RETFIT - assert(op2 == nullptr); - return 0; - } - - if (op2 == nullptr) - { - gtSetEvalOrderMinOpts(op1); - return 1; - } - - level = gtSetEvalOrderMinOpts(op1); - unsigned levelOp2 = gtSetEvalOrderMinOpts(op2); - - bool allowSwap = true; - // TODO: Introduce a function to check whether we can swap the order of its operands or not. - switch (tree->OperGet()) - { - case GT_COMMA: - case GT_BOUNDS_CHECK: - case GT_INTRINSIC: - case GT_QMARK: - case GT_COLON: - // We're not going to swap operands in these - allowSwap = false; - break; - - case GT_STORE_BLK: - case GT_STOREIND: - SetIndirectStoreEvalOrder(this, tree->AsIndir(), &allowSwap); - break; - - default: - break; - } - - const bool shouldSwap = tree->IsReverseOp() ? level > levelOp2 : level < levelOp2; - if (shouldSwap && allowSwap) - { - // Can we swap the order by commuting the operands? - const bool canSwap = tree->IsReverseOp() ? gtCanSwapOrder(op2, op1) : gtCanSwapOrder(op1, op2); - if (canSwap) - { - if (tree->OperIsCmpCompare()) - { - genTreeOps oper = tree->OperGet(); - if (GenTree::SwapRelop(oper) != oper) - { - tree->SetOper(GenTree::SwapRelop(oper)); - } - std::swap(tree->AsOp()->gtOp1, tree->AsOp()->gtOp2); - } - else if (tree->OperIsCommutative()) - { - std::swap(tree->AsOp()->gtOp1, tree->AsOp()->gtOp2); - } - else - { - // Mark the operand's evaluation order to be swapped. - tree->gtFlags ^= GTF_REVERSE_OPS; - } - } - } - - // Swap the level counts - if (tree->IsReverseOp()) - { - std::swap(level, levelOp2); - } - - // Compute the sethi number for this binary operator - if (level < 1) - { - level = levelOp2; - } - else if (level == levelOp2) - { - level++; - } - } - else if (tree->IsCall()) - { - // We ignore late args - they don't bring any noticeable benefits - // according to asmdiffs/tpdiff - for (CallArg& arg : tree->AsCall()->gtArgs.EarlyArgs()) - { - gtSetEvalOrderMinOpts(arg.GetEarlyNode()); - } - level = 3; - } -#if defined(FEATURE_HW_INTRINSICS) - else if (tree->OperIsHWIntrinsic()) - { - return gtSetMultiOpOrder(tree->AsMultiOp()); - } -#endif // FEATURE_HW_INTRINSICS - - // NOTE: we skip many operators here in order to maintain a good trade-off between CQ and TP. - - return level; -} - -//------------------------------------------------------------------------ -// gtMayHaveStoreInterference: Check if two trees may interfere because of a -// store in one of the trees. -// -// Parameters: -// treeWithStores - Tree that may have stores in it -// tree - Tree that may be reading from a local stored to in "treeWithStores" -// -// Returns: -// False if there is no interference. Returns true if there is any GT_LCL_VAR -// or GT_LCL_FLD in "tree" whose value depends on a local stored in -// "treeWithStores". May also return true in cases without interference if -// the trees are too large and the function runs out of budget. -// -bool Compiler::gtMayHaveStoreInterference(GenTree* treeWithStores, GenTree* tree) -{ - if (((treeWithStores->gtFlags & GTF_ASG) == 0) || tree->IsInvariant()) - { - return false; - } - - class Visitor final : public GenTreeVisitor - { - GenTree* m_readTree; - unsigned m_numStoresChecked = 0; - - public: - enum - { - DoPreOrder = true, - }; - - Visitor(Compiler* compiler, GenTree* readTree) - : GenTreeVisitor(compiler) - , m_readTree(readTree) - { - } - - Compiler::fgWalkResult PreOrderVisit(GenTree** use, GenTree* user) - { - GenTree* node = *use; - if ((node->gtFlags & GTF_ASG) == 0) - { - return WALK_SKIP_SUBTREES; - } - - if (node->OperIsLocalStore()) - { - // Check up to 8 stores before we bail with a conservative - // answer. Avoids quadratic behavior in case we have a large - // number of stores (e.g. created by physical promotion or by - // call args morphing). - if ((m_numStoresChecked >= 8) || - m_compiler->gtTreeHasLocalRead(m_readTree, node->AsLclVarCommon()->GetLclNum())) - { - return WALK_ABORT; - } - - m_numStoresChecked++; - } - - return WALK_CONTINUE; - } - }; - - Visitor visitor(this, tree); - return visitor.WalkTree(&treeWithStores, nullptr) == WALK_ABORT; -} - -//------------------------------------------------------------------------ -// gtTreeHasLocalRead: Check if a tree has a read of the specified local, -// taking promotion into account. -// -// Parameters: -// tree - The tree to check. -// lclNum - The local to look for. -// -// Returns: -// True if there is any GT_LCL_VAR or GT_LCL_FLD node whose value depends on -// "lclNum". -// -bool Compiler::gtTreeHasLocalRead(GenTree* tree, unsigned lclNum) -{ - class Visitor final : public GenTreeVisitor - { - public: - enum - { - DoPreOrder = true, - DoLclVarsOnly = true, - }; - - unsigned m_lclNum; - LclVarDsc* m_lclDsc; - - Visitor(Compiler* compiler, unsigned lclNum) - : GenTreeVisitor(compiler) - , m_lclNum(lclNum) - { - m_lclDsc = compiler->lvaGetDesc(lclNum); - } - - Compiler::fgWalkResult PreOrderVisit(GenTree** use, GenTree* user) - { - GenTree* node = *use; - - if (node->OperIsLocalRead()) - { - if (node->AsLclVarCommon()->GetLclNum() == m_lclNum) - { - return WALK_ABORT; - } - - if (m_lclDsc->lvIsStructField && (node->AsLclVarCommon()->GetLclNum() == m_lclDsc->lvParentLcl)) - { - return WALK_ABORT; - } - - if (m_lclDsc->lvPromoted && (node->AsLclVarCommon()->GetLclNum() >= m_lclDsc->lvFieldLclStart) && - (node->AsLclVarCommon()->GetLclNum() < m_lclDsc->lvFieldLclStart + m_lclDsc->lvFieldCnt)) - { - return WALK_ABORT; - } - } - - return WALK_CONTINUE; - } - }; - - Visitor visitor(this, lclNum); - return visitor.WalkTree(&tree, nullptr) == WALK_ABORT; -} - -#ifdef DEBUG -bool GenTree::OperSupportsReverseOpEvalOrder(Compiler* comp) const -{ - if (OperIsBinary()) - { - if ((AsOp()->gtGetOp1() == nullptr) || (AsOp()->gtGetOp2() == nullptr)) - { - return false; - } - if (OperIs(GT_COMMA, GT_BOUNDS_CHECK)) - { - return false; - } - if (OperIs(GT_INTRINSIC)) - { - return !comp->IsIntrinsicImplementedByUserCall(AsIntrinsic()->gtIntrinsicName); - } - return true; - } -#if defined(FEATURE_SIMD) || defined(FEATURE_HW_INTRINSICS) - if (OperIsMultiOp()) - { - return (AsMultiOp()->GetOperandCount() == 2) && !AsMultiOp()->IsUserCall(); - } -#endif // FEATURE_SIMD || FEATURE_HW_INTRINSICS - return false; -} -#endif // DEBUG - -/***************************************************************************** - * - * If the given tree is an integer constant that can be used - * in a scaled index address mode as a multiplier (e.g. "[4*index]"), then return - * the scale factor: 2, 4, or 8. Otherwise, return 0. Note that we never return 1, - * to match the behavior of GetScaleIndexShf(). - */ - -unsigned GenTree::GetScaleIndexMul() -{ - if (IsCnsIntOrI() && jitIsScaleIndexMul(AsIntConCommon()->IconValue()) && AsIntConCommon()->IconValue() != 1) - { - return (unsigned)AsIntConCommon()->IconValue(); - } - - return 0; -} - -/***************************************************************************** - * - * If the given tree is the right-hand side of a left shift (that is, - * 'y' in the tree 'x' << 'y'), and it is an integer constant that can be used - * in a scaled index address mode as a multiplier (e.g. "[4*index]"), then return - * the scale factor: 2, 4, or 8. Otherwise, return 0. - */ - -unsigned GenTree::GetScaleIndexShf() -{ - if (IsCnsIntOrI() && jitIsScaleIndexShift(AsIntConCommon()->IconValue())) - { - return (unsigned)(1 << AsIntConCommon()->IconValue()); - } - - return 0; -} - -/***************************************************************************** - * - * If the given tree is a scaled index (i.e. "op * 4" or "op << 2"), returns - * the multiplier: 2, 4, or 8; otherwise returns 0. Note that "1" is never - * returned. - */ - -unsigned GenTree::GetScaledIndex() -{ - if (AsOp()->gtOp1->IsCnsIntOrI()) - { - return 0; - } - - switch (gtOper) - { - case GT_MUL: - return AsOp()->gtOp2->GetScaleIndexMul(); - - case GT_LSH: - return AsOp()->gtOp2->GetScaleIndexShf(); - - default: - assert(!"GenTree::GetScaledIndex() called with illegal gtOper"); - break; - } - - return 0; -} - -//------------------------------------------------------------------------ -// TryGetUse: Get the use edge for an operand of this tree. -// -// Arguments: -// operand - the node to find the use for -// pUse - [out] parameter for the use -// -// Return Value: -// Whether "operand" is a child of this node. If it is, "*pUse" is set, -// allowing for the replacement of "operand" with some other node. -// -bool GenTree::TryGetUse(GenTree* operand, GenTree*** pUse) -{ - assert(operand != nullptr); - assert(pUse != nullptr); - - switch (OperGet()) - { - // Leaf nodes - case GT_LCL_VAR: - case GT_LCL_FLD: - case GT_LCL_ADDR: - case GT_CATCH_ARG: - case GT_LABEL: - case GT_FTN_ADDR: - case GT_RET_EXPR: - case GT_CNS_INT: - case GT_CNS_LNG: - case GT_CNS_DBL: - case GT_CNS_STR: -#if defined(FEATURE_SIMD) - case GT_CNS_VEC: -#endif // FEATURE_SIMD -#if defined(FEATURE_MASKED_HW_INTRINSICS) - case GT_CNS_MSK: -#endif // FEATURE_MASKED_HW_INTRINSICS - case GT_MEMORYBARRIER: - case GT_JMP: - case GT_JCC: - case GT_SETCC: - case GT_NO_OP: - case GT_START_NONGC: - case GT_START_PREEMPTGC: - case GT_PROF_HOOK: -#if defined(FEATURE_EH_WINDOWS_X86) - case GT_END_LFIN: -#endif // FEATURE_EH_WINDOWS_X86 - case GT_PHI_ARG: - case GT_JMPTABLE: - case GT_PHYSREG: - case GT_EMITNOP: - case GT_PINVOKE_PROLOG: - case GT_PINVOKE_EPILOG: - case GT_IL_OFFSET: - case GT_NOP: - case GT_SWIFT_ERROR: - return false; - - // Standard unary operators - case GT_STORE_LCL_VAR: - case GT_STORE_LCL_FLD: - case GT_NOT: - case GT_NEG: - case GT_COPY: - case GT_RELOAD: - case GT_ARR_LENGTH: - case GT_MDARR_LENGTH: - case GT_MDARR_LOWER_BOUND: - case GT_CAST: - case GT_BITCAST: - case GT_CKFINITE: - case GT_LCLHEAP: - case GT_IND: - case GT_BLK: - case GT_BOX: - case GT_ALLOCOBJ: - case GT_RUNTIMELOOKUP: - case GT_ARR_ADDR: - case GT_INIT_VAL: - case GT_JTRUE: - case GT_SWITCH: - case GT_NULLCHECK: - case GT_PUTARG_REG: - case GT_PUTARG_STK: - case GT_RETURNTRAP: - case GT_RETURN: - case GT_RETFILT: - case GT_BSWAP: - case GT_BSWAP16: - case GT_KEEPALIVE: - case GT_INC_SATURATE: - if (operand == this->AsUnOp()->gtOp1) - { - *pUse = &this->AsUnOp()->gtOp1; - return true; - } - return false; - -// Variadic nodes -#if FEATURE_ARG_SPLIT - case GT_PUTARG_SPLIT: - if (this->AsUnOp()->gtOp1->gtOper == GT_FIELD_LIST) - { - return this->AsUnOp()->gtOp1->TryGetUse(operand, pUse); - } - if (operand == this->AsUnOp()->gtOp1) - { - *pUse = &this->AsUnOp()->gtOp1; - return true; - } - return false; -#endif // FEATURE_ARG_SPLIT - -#if defined(FEATURE_HW_INTRINSICS) - case GT_HWINTRINSIC: - for (GenTree** opUse : this->AsMultiOp()->UseEdges()) - { - if (*opUse == operand) - { - *pUse = opUse; - return true; - } - } - return false; -#endif // FEATURE_HW_INTRINSICS - - // Special nodes - case GT_PHI: - for (GenTreePhi::Use& phiUse : AsPhi()->Uses()) - { - if (phiUse.GetNode() == operand) - { - *pUse = &phiUse.NodeRef(); - return true; - } - } - return false; - - case GT_FIELD_LIST: - for (GenTreeFieldList::Use& fieldUse : AsFieldList()->Uses()) - { - if (fieldUse.GetNode() == operand) - { - *pUse = &fieldUse.NodeRef(); - return true; - } - } - return false; - - case GT_CMPXCHG: - { - GenTreeCmpXchg* const cmpXchg = this->AsCmpXchg(); - if (operand == cmpXchg->Addr()) - { - *pUse = &cmpXchg->Addr(); - return true; - } - if (operand == cmpXchg->Data()) - { - *pUse = &cmpXchg->Data(); - return true; - } - if (operand == cmpXchg->Comparand()) - { - *pUse = &cmpXchg->Comparand(); - return true; - } - return false; - } - - case GT_ARR_ELEM: - { - GenTreeArrElem* const arrElem = this->AsArrElem(); - if (operand == arrElem->gtArrObj) - { - *pUse = &arrElem->gtArrObj; - return true; - } - for (unsigned i = 0; i < arrElem->gtArrRank; i++) - { - if (operand == arrElem->gtArrInds[i]) - { - *pUse = &arrElem->gtArrInds[i]; - return true; - } - } - return false; - } - - case GT_CALL: - { - GenTreeCall* const call = this->AsCall(); - if (operand == call->gtControlExpr) - { - *pUse = &call->gtControlExpr; - return true; - } - if (call->gtCallType == CT_INDIRECT) - { - if (operand == call->gtCallCookie) - { - *pUse = &call->gtCallCookie; - return true; - } - if (operand == call->gtCallAddr) - { - *pUse = &call->gtCallAddr; - return true; - } - } - for (CallArg& arg : call->gtArgs.Args()) - { - if (arg.GetEarlyNode() == operand) - { - *pUse = &arg.EarlyNodeRef(); - return true; - } - if (arg.GetLateNode() == operand) - { - *pUse = &arg.LateNodeRef(); - return true; - } - } - return false; - } -#ifdef TARGET_ARM64 - case GT_SELECT_NEG: - case GT_SELECT_INV: - case GT_SELECT_INC: -#endif - case GT_SELECT: - { - GenTreeConditional* const conditional = this->AsConditional(); - if (operand == conditional->gtCond) - { - *pUse = &conditional->gtCond; - return true; - } - if (operand == conditional->gtOp1) - { - *pUse = &conditional->gtOp1; - return true; - } - if (operand == conditional->gtOp2) - { - *pUse = &conditional->gtOp2; - return true; - } - return false; - } - - // Binary nodes - default: - assert(this->OperIsBinary()); - return TryGetUseBinOp(operand, pUse); - } -} - -bool GenTree::TryGetUseBinOp(GenTree* operand, GenTree*** pUse) -{ - assert(operand != nullptr); - assert(pUse != nullptr); - assert(this->OperIsBinary()); - - GenTreeOp* const binOp = this->AsOp(); - if (operand == binOp->gtOp1) - { - *pUse = &binOp->gtOp1; - return true; - } - if (operand == binOp->gtOp2) - { - *pUse = &binOp->gtOp2; - return true; - } - return false; -} - -//------------------------------------------------------------------------ -// GenTree::ReplaceOperand: -// Replace a given operand to this node with a new operand. If the -// current node is a call node, this will also update the call -// argument table if necessary. -// -// Arguments: -// useEdge - the use edge that points to the operand to be replaced. -// replacement - the replacement node. -// -void GenTree::ReplaceOperand(GenTree** useEdge, GenTree* replacement) -{ - assert(useEdge != nullptr); - assert(replacement != nullptr); - assert(TryGetUse(*useEdge, &useEdge)); - *useEdge = replacement; -} - -//------------------------------------------------------------------------ -// gtGetParent: Get the parent of this node, and optionally capture the -// pointer to the child so that it can be modified. -// -// Arguments: -// pUse - A pointer to a GenTree** (yes, that's three -// levels, i.e. GenTree ***), which if non-null, -// will be set to point to the field in the parent -// that points to this node. -// -// Return value -// The parent of this node. -// -// Notes: -// This requires that the execution order must be defined (i.e. gtSetEvalOrder() has been called). -// To enable the child to be replaced, it accepts an argument, "pUse", that, if non-null, -// will be set to point to the child pointer in the parent that points to this node. -// -GenTree* GenTree::gtGetParent(GenTree*** pUse) -{ - // Find the parent node; it must be after this node in the execution order. - GenTree* user; - GenTree** use = nullptr; - for (user = gtNext; user != nullptr; user = user->gtNext) - { - if (user->TryGetUse(this, &use)) - { - break; - } - } - - if (pUse != nullptr) - { - *pUse = use; - } - - return user; -} - -//------------------------------------------------------------------------------ -// OperRequiresAsgFlag : Check whether the operation requires GTF_ASG flag regardless -// of the children's flags. -// - -bool GenTree::OperRequiresAsgFlag() const -{ - switch (OperGet()) - { - case GT_STORE_LCL_VAR: - case GT_STORE_LCL_FLD: - case GT_STOREIND: - case GT_STORE_BLK: - case GT_XADD: - case GT_XORR: - case GT_XAND: - case GT_XCHG: - case GT_LOCKADD: - case GT_CMPXCHG: - case GT_MEMORYBARRIER: - return true; - - // If the call has return buffer argument, it produced a definition and hence - // should be marked with GTF_ASG. - case GT_CALL: - return AsCall()->IsOptimizingRetBufAsLocal(); - -#ifdef FEATURE_HW_INTRINSICS - case GT_HWINTRINSIC: - return AsHWIntrinsic()->OperRequiresAsgFlag(); -#endif // FEATURE_HW_INTRINSICS - - default: - assert(!OperIsStore() && !OperIsAtomicOp()); - return false; - } -} - -//------------------------------------------------------------------------------ -// OperRequiresCallFlag : Check whether the operation requires GTF_CALL flag regardless -// of the children's flags. -// -bool GenTree::OperRequiresCallFlag(Compiler* comp) const -{ - switch (gtOper) - { - case GT_CALL: - return true; - - case GT_KEEPALIVE: - return true; - - case GT_SWIFT_ERROR: - return true; - - case GT_INTRINSIC: - return comp->IsIntrinsicImplementedByUserCall(this->AsIntrinsic()->gtIntrinsicName); - -#if defined(FEATURE_HW_INTRINSICS) - case GT_HWINTRINSIC: - { - return AsHWIntrinsic()->OperRequiresCallFlag(); - } -#endif // FEATURE_HW_INTRINSICS - -#if FEATURE_FIXED_OUT_ARGS && !defined(TARGET_64BIT) - case GT_LSH: - case GT_RSH: - case GT_RSZ: - - // Variable shifts of a long end up being helper calls, so mark the tree as such in morph. - // This is potentially too conservative, since they'll get treated as having side effects. - // It is important to mark them as calls so if they are part of an argument list, - // they will get sorted and processed properly (for example, it is important to handle - // all nested calls before putting struct arguments in the argument registers). We - // could mark the trees just before argument processing, but it would require a full - // tree walk of the argument tree, so we just do it when morphing, instead, even though we'll - // mark non-argument trees (that will still get converted to calls, anyway). - return (this->TypeGet() == TYP_LONG) && (gtGetOp2()->OperGet() != GT_CNS_INT); -#endif // FEATURE_FIXED_OUT_ARGS && !TARGET_64BIT - - default: - return false; - } -} - -//------------------------------------------------------------------------ -// IndirMayFault: May this indirection-like node throw an NRE? -// -// Arguments: -// compiler - the compiler instance -// -// Return Value: -// Whether this node's address may be null. -// -bool GenTree::IndirMayFault(Compiler* compiler) -{ - assert(OperIsIndirOrArrMetaData()); - return ((gtFlags & GTF_IND_NONFAULTING) == 0) && compiler->fgAddrCouldBeNull(GetIndirOrArrMetaDataAddr()); -} - -//------------------------------------------------------------------------------ -// OperIsImplicitIndir : Check whether the operation contains an implicit indirection. -// -// Arguments: -// this - a GenTree node -// -// Return Value: -// True if the given node contains an implicit indirection -// -// Note that for the [HW]INTRINSIC nodes we have to examine the -// details of the node to determine its result. -// -bool GenTree::OperIsImplicitIndir() const -{ - switch (gtOper) - { - case GT_LOCKADD: - case GT_XORR: - case GT_XAND: - case GT_XADD: - case GT_XCHG: - case GT_CMPXCHG: - case GT_BLK: - case GT_STORE_BLK: - case GT_BOX: - case GT_ARR_ELEM: - case GT_ARR_LENGTH: - case GT_MDARR_LENGTH: - case GT_MDARR_LOWER_BOUND: - return true; - case GT_INTRINSIC: - return AsIntrinsic()->gtIntrinsicName == NI_System_Object_GetType; -#ifdef FEATURE_HW_INTRINSICS - case GT_HWINTRINSIC: - { - return AsHWIntrinsic()->OperIsMemoryLoadOrStore(); - } -#endif // FEATURE_HW_INTRINSICS - default: - return false; - } -} - -//------------------------------------------------------------------------------ -// OperExceptions: Get exception set this tree may throw. -// -// Arguments: -// comp - Compiler instance -// -// Return Value: -// A bit set of exceptions this tree may throw. -// -// Remarks: -// Should not be used on calls given that we can say nothing precise about -// those. -// -ExceptionSetFlags GenTree::OperExceptions(Compiler* comp) -{ - switch (gtOper) - { - case GT_ADD: - case GT_SUB: - case GT_MUL: - case GT_CAST: -#ifndef TARGET_64BIT - case GT_ADD_HI: - case GT_SUB_HI: -#endif // !TARGET_64BIT - return gtOverflow() ? ExceptionSetFlags::OverflowException : ExceptionSetFlags::None; - - case GT_MOD: - case GT_DIV: - case GT_UMOD: - case GT_UDIV: - { - if (varTypeIsFloating(this->TypeGet())) - { - return ExceptionSetFlags::None; - } - - ExceptionSetFlags exSetFlags = ExceptionSetFlags::None; - - if (!(this->gtFlags & GTF_DIV_MOD_NO_BY_ZERO) && !this->gtGetOp2()->gtSkipReloadOrCopy()->IsNeverZero()) - { - exSetFlags |= ExceptionSetFlags::DivideByZeroException; - } - - if (this->OperIs(GT_DIV, GT_MOD) && this->CanDivOrModPossiblyOverflow(comp)) - { - exSetFlags |= ExceptionSetFlags::ArithmeticException; - } - - return exSetFlags; - } - - case GT_INTRINSIC: - // If this is an intrinsic that represents the object.GetType(), it can throw an NullReferenceException. - // Currently, this is the only intrinsic that can throw an exception. - if (AsIntrinsic()->gtIntrinsicName == NI_System_Object_GetType) - { - return ExceptionSetFlags::NullReferenceException; - } - - return ExceptionSetFlags::None; - - case GT_CALL: - assert(!"Unexpected GT_CALL in OperExceptions"); - return ExceptionSetFlags::All; - - case GT_LOCKADD: - case GT_XAND: - case GT_XORR: - case GT_XADD: - case GT_XCHG: - case GT_CMPXCHG: - case GT_IND: - case GT_STOREIND: - case GT_BLK: - case GT_NULLCHECK: - case GT_STORE_BLK: - case GT_ARR_LENGTH: - case GT_MDARR_LENGTH: - case GT_MDARR_LOWER_BOUND: - return IndirMayFault(comp) ? ExceptionSetFlags::NullReferenceException : ExceptionSetFlags::None; - - case GT_ARR_ELEM: - if (comp->fgAddrCouldBeNull(this->AsArrElem()->gtArrObj)) - { - return ExceptionSetFlags::NullReferenceException | ExceptionSetFlags::IndexOutOfRangeException; - } - - return ExceptionSetFlags::IndexOutOfRangeException; - - case GT_FIELD_ADDR: - if (AsFieldAddr()->IsInstance() && comp->fgAddrCouldBeNull(AsFieldAddr()->GetFldObj())) - { - return ExceptionSetFlags::NullReferenceException; - } - - return ExceptionSetFlags::None; - - case GT_BOUNDS_CHECK: - return ExceptionSetFlags::IndexOutOfRangeException; - - case GT_INDEX_ADDR: - return ExceptionSetFlags::NullReferenceException | ExceptionSetFlags::IndexOutOfRangeException; - - case GT_CKFINITE: - return ExceptionSetFlags::ArithmeticException; - - case GT_LCLHEAP: - return ExceptionSetFlags::StackOverflowException; - -#ifdef FEATURE_HW_INTRINSICS - case GT_HWINTRINSIC: - { - assert((gtFlags & GTF_HW_USER_CALL) == 0); - - GenTreeHWIntrinsic* hwIntrinsicNode = this->AsHWIntrinsic(); - - if (hwIntrinsicNode->OperIsMemoryLoadOrStore()) - { - // TODO-CQ: We should use comp->fgAddrCouldBeNull on the address operand - // to determine if this can actually produce an NRE or not - - return ExceptionSetFlags::NullReferenceException; - } - - return ExceptionSetFlags::None; - } -#endif // FEATURE_HW_INTRINSICS - - default: - assert(!OperMayOverflow() && !OperIsIndirOrArrMetaData()); - return ExceptionSetFlags::None; - } -} - -//------------------------------------------------------------------------------ -// OperMayThrow : Check whether the operation may throw. -// -// -// Arguments: -// comp - Compiler instance -// -// Return Value: -// True if the given operator may cause an exception -// -bool GenTree::OperMayThrow(Compiler* comp) -{ - if (OperIs(GT_CALL)) - { - CorInfoHelpFunc helper; - helper = comp->eeGetHelperNum(this->AsCall()->gtCallMethHnd); - return ((helper == CORINFO_HELP_UNDEF) || !comp->s_helperCallProperties.NoThrow(helper)); - } -#ifdef FEATURE_HW_INTRINSICS - else if (OperIsHWIntrinsic()) - { - if ((gtFlags & GTF_HW_USER_CALL) != 0) - { - return true; - } - } -#endif // FEATURE_HW_INTRINSICS - - return OperExceptions(comp) != ExceptionSetFlags::None; -} - -//------------------------------------------------------------------------------ -// OperRequiresGlobRefFlag : Check whether the operation requires GTF_GLOB_REF -// flag regardless of the children's flags. -// -// Arguments: -// comp - Compiler instance -// -// Return Value: -// True if the given operator requires GTF_GLOB_REF -// -// Remarks: -// Globally visible stores and loads, as well as some equivalently modeled -// operations, require the GLOB_REF flag to be set on the node. -// -// This function is only valid after local morph when we know which locals -// are address exposed, and the flag in gtFlags is only kept valid after -// morph has run. Before local morph the property can be conservatively -// approximated for locals with lvHasLdAddrOp. -// -bool GenTree::OperRequiresGlobRefFlag(Compiler* comp) const -{ - switch (OperGet()) - { - case GT_LCL_VAR: - case GT_LCL_FLD: - case GT_STORE_LCL_VAR: - case GT_STORE_LCL_FLD: - return comp->lvaGetDesc(AsLclVarCommon())->IsAddressExposed(); - - case GT_IND: - case GT_BLK: - if (AsIndir()->IsInvariantLoad()) - { - return false; - } - FALLTHROUGH; - - case GT_STOREIND: - case GT_STORE_BLK: - case GT_XADD: - case GT_XORR: - case GT_XAND: - case GT_XCHG: - case GT_LOCKADD: - case GT_CMPXCHG: - case GT_MEMORYBARRIER: - case GT_KEEPALIVE: - case GT_SWIFT_ERROR: - return true; - - case GT_CALL: - return AsCall()->HasSideEffects(comp, /* ignoreExceptions */ true); - - case GT_ALLOCOBJ: - return AsAllocObj()->gtHelperHasSideEffects; - -#if defined(FEATURE_HW_INTRINSICS) - case GT_HWINTRINSIC: - return AsHWIntrinsic()->OperRequiresGlobRefFlag(); -#endif // FEATURE_HW_INTRINSICS - - default: - assert(!OperRequiresCallFlag(comp) || OperIs(GT_INTRINSIC)); - assert((!OperIsIndir() || OperIs(GT_NULLCHECK)) && !OperRequiresAsgFlag()); - return false; - } -} - -//------------------------------------------------------------------------------ -// OperSupportsOrderingSideEffect : Check whether the operation supports the -// GTF_ORDER_SIDEEFF flag. -// -// Return Value: -// True if the given operator supports GTF_ORDER_SIDEEFF. -// -// Remarks: -// A node will still have this flag set if an operand has it set, even if the -// parent does not support it. This situation indicates that reordering the -// parent may be ok as long as it does not break ordering dependencies of the -// operand. -// -bool GenTree::OperSupportsOrderingSideEffect() const -{ - if (TypeIs(TYP_BYREF)) - { - // Forming byrefs may only be legal due to previous checks. - return true; - } - - switch (OperGet()) - { - case GT_BOUNDS_CHECK: - case GT_IND: - case GT_BLK: - case GT_STOREIND: - case GT_NULLCHECK: - case GT_STORE_BLK: - case GT_XADD: - case GT_XORR: - case GT_XAND: - case GT_XCHG: - case GT_LOCKADD: - case GT_CMPXCHG: - case GT_MEMORYBARRIER: - case GT_CATCH_ARG: - case GT_SWIFT_ERROR: - return true; - default: - return false; - } -} - -//------------------------------------------------------------------------------ -// OperEffects: Compute effect flags that are relevant to this node only, -// excluding its children. -// -// Arguments: -// comp - Compiler instance -// -// Return Value: -// The effect flags. -// -GenTreeFlags GenTree::OperEffects(Compiler* comp) -{ - GenTreeFlags flags = gtFlags & GTF_ALL_EFFECT; - - if (((flags & GTF_ASG) != 0) && !OperRequiresAsgFlag()) - { - flags &= ~GTF_ASG; - } - - if (((flags & GTF_CALL) != 0) && !OperRequiresCallFlag(comp)) - { - flags &= ~GTF_CALL; - } - - if (((flags & GTF_EXCEPT) != 0) && !OperMayThrow(comp)) - { - flags &= ~GTF_EXCEPT; - } - - if (((flags & GTF_GLOB_REF) != 0) && !OperRequiresGlobRefFlag(comp)) - { - flags &= ~GTF_GLOB_REF; - } - - if (((flags & GTF_ORDER_SIDEEFF) != 0) && !OperSupportsOrderingSideEffect()) - { - flags &= ~GTF_ORDER_SIDEEFF; - } - - return flags; -} - -//----------------------------------------------------------------------------------- -// GetFieldCount: Return the register count for a multi-reg lclVar. -// -// Arguments: -// compiler - the current Compiler instance. -// -// Return Value: -// Returns the number of registers defined by this node. -// -// Notes: -// This must be a multireg lclVar. -// -unsigned int GenTreeLclVar::GetFieldCount(Compiler* compiler) const -{ - assert(IsMultiReg()); - LclVarDsc* varDsc = compiler->lvaGetDesc(GetLclNum()); - return varDsc->lvFieldCnt; -} - -//----------------------------------------------------------------------------------- -// GetFieldTypeByIndex: Get a specific register's type, based on regIndex, that is produced -// by this multi-reg node. -// -// Arguments: -// compiler - the current Compiler instance. -// idx - which register type to return. -// -// Return Value: -// The register type assigned to this index for this node. -// -// Notes: -// This must be a multireg lclVar and 'regIndex' must be a valid index for this node. -// -var_types GenTreeLclVar::GetFieldTypeByIndex(Compiler* compiler, unsigned idx) -{ - assert(IsMultiReg()); - LclVarDsc* varDsc = compiler->lvaGetDesc(GetLclNum()); - LclVarDsc* fieldVarDsc = compiler->lvaGetDesc(varDsc->lvFieldLclStart + idx); - assert(fieldVarDsc->TypeGet() != TYP_STRUCT); // Don't expect struct fields. - return fieldVarDsc->TypeGet(); -} - -#if DEBUGGABLE_GENTREE -// static -GenTree::VtablePtr GenTree::s_vtablesForOpers[] = {nullptr}; -GenTree::VtablePtr GenTree::s_vtableForOp = nullptr; - -GenTree::VtablePtr GenTree::GetVtableForOper(genTreeOps oper) -{ - noway_assert(oper < GT_COUNT); - - // First, check a cache. - - if (s_vtablesForOpers[oper] != nullptr) - { - return s_vtablesForOpers[oper]; - } - - // Otherwise, look up the correct vtable entry. Note that we want the most derived GenTree subtype - // for an oper. E.g., GT_LCL_VAR is defined in GTSTRUCT_3 as GenTreeLclVar and in GTSTRUCT_N as - // GenTreeLclVarCommon. We want the GenTreeLclVar vtable, since nothing should actually be - // instantiated as a GenTreeLclVarCommon. - - VtablePtr res = nullptr; - switch (oper) - { - - // clang-format off - -#define GTSTRUCT_0(nm, tag) /*handle explicitly*/ -#define GTSTRUCT_1(nm, tag) \ - case tag: \ - { \ - GenTree##nm gt; \ - res = *reinterpret_cast(>); \ - } \ - break; -#define GTSTRUCT_2(nm, tag, tag2) \ - case tag: \ - case tag2: \ - { \ - GenTree##nm gt; \ - res = *reinterpret_cast(>); \ - } \ - break; -#define GTSTRUCT_3(nm, tag, tag2, tag3) \ - case tag: \ - case tag2: \ - case tag3: \ - { \ - GenTree##nm gt; \ - res = *reinterpret_cast(>); \ - } \ - break; -#define GTSTRUCT_4(nm, tag, tag2, tag3, tag4) \ - case tag: \ - case tag2: \ - case tag3: \ - case tag4: \ - { \ - GenTree##nm gt; \ - res = *reinterpret_cast(>); \ - } \ - break; -#define GTSTRUCT_N(nm, ...) /*handle explicitly*/ -#define GTSTRUCT_2_SPECIAL(nm, tag, tag2) /*handle explicitly*/ -#define GTSTRUCT_3_SPECIAL(nm, tag, tag2, tag3) /*handle explicitly*/ -#include "gtstructs.h" - - // clang-format on - - // Handle the special cases. - // The following opers are in GTSTRUCT_N but no other place (namely, no subtypes). - case GT_STORE_BLK: - case GT_BLK: - { - GenTreeBlk gt; - res = *reinterpret_cast(>); - } - break; - - case GT_IND: - case GT_NULLCHECK: - { - GenTreeIndir gt; - res = *reinterpret_cast(>); - } - break; - - // We don't need to handle GTSTRUCT_N for LclVarCommon, since all those allowed opers are specified - // in their proper subtype. Similarly for GenTreeIndir. - - default: - { - // Should be unary or binary op. - if (s_vtableForOp == nullptr) - { - unsigned opKind = OperKind(oper); - assert(!IsExOp(opKind)); - assert(OperIsSimple(oper) || OperIsLeaf(oper)); - // Need to provide non-null operands. - GenTreeIntCon dummyOp(TYP_INT, 0); - GenTreeOp gt(oper, TYP_INT, &dummyOp, ((opKind & GTK_UNOP) ? nullptr : &dummyOp)); - s_vtableForOp = *reinterpret_cast(>); - } - res = s_vtableForOp; - break; - } - } - s_vtablesForOpers[oper] = res; - return res; -} - -void GenTree::SetVtableForOper(genTreeOps oper) -{ - *reinterpret_cast(this) = GetVtableForOper(oper); -} -#endif // DEBUGGABLE_GENTREE - -GenTreeOp* Compiler::gtNewOperNode(genTreeOps oper, var_types type, GenTree* op1, GenTree* op2) -{ - assert(op1 != nullptr); - assert(op2 != nullptr); - - // We should not be allocating nodes that extend GenTreeOp with this; - // should call the appropriate constructor for the extended type. - assert(!GenTree::IsExOp(GenTree::OperKind(oper))); - - GenTreeOp* node = new (this, oper) GenTreeOp(oper, type, op1, op2); - - return node; -} - -GenTreeCC* Compiler::gtNewCC(genTreeOps oper, var_types type, GenCondition cond) -{ - GenTreeCC* node = new (this, oper) GenTreeCC(oper, type, cond); - return node; -} - -GenTreeOpCC* Compiler::gtNewOperCC(genTreeOps oper, var_types type, GenCondition cond, GenTree* op1, GenTree* op2) -{ - GenTreeOpCC* node = new (this, oper) GenTreeOpCC(oper, type, cond, op1, op2); - return node; -} - -GenTreeColon* Compiler::gtNewColonNode(var_types type, GenTree* thenNode, GenTree* elseNode) -{ - return new (this, GT_COLON) GenTreeColon(type, thenNode, elseNode); -} - -GenTreeQmark* Compiler::gtNewQmarkNode(var_types type, GenTree* cond, GenTreeColon* colon) -{ - compQmarkUsed = true; - GenTreeQmark* result = new (this, GT_QMARK) GenTreeQmark(type, cond, colon); - assert(!compQmarkRationalized && "QMARKs are illegal to create after QMARK-rationalization"); - return result; -} - -GenTreeIntCon* Compiler::gtNewIconNode(ssize_t value, var_types type) -{ - assert(genActualType(type) == type); - return new (this, GT_CNS_INT) GenTreeIntCon(type, value); -} - -GenTreeIntCon* Compiler::gtNewIconNodeWithVN(Compiler* comp, ssize_t value, var_types type) -{ - assert(genActualType(type) == type); - GenTreeIntCon* cns = new (this, GT_CNS_INT) GenTreeIntCon(type, value); - comp->fgUpdateConstTreeValueNumber(cns); - return cns; -} - -GenTreeIntCon* Compiler::gtNewIconNode(unsigned fieldOffset, FieldSeq* fieldSeq) -{ - return new (this, GT_CNS_INT) GenTreeIntCon(TYP_I_IMPL, static_cast(fieldOffset), fieldSeq); -} - -GenTreeIntCon* Compiler::gtNewNull() -{ - return gtNewIconNode(0, TYP_REF); -} - -GenTreeIntCon* Compiler::gtNewTrue() -{ - return gtNewIconNode(1, TYP_INT); -} - -GenTreeIntCon* Compiler::gtNewFalse() -{ - return gtNewIconNode(0, TYP_INT); -} - -// return a new node representing the value in a physical register -GenTree* Compiler::gtNewPhysRegNode(regNumber reg, var_types type) -{ -#ifdef TARGET_ARM64 - assert(genIsValidIntReg(reg) || (reg == REG_SPBASE) || (reg == REG_FFR)); -#else - assert(genIsValidIntReg(reg) || (reg == REG_SPBASE)); -#endif - GenTree* result = new (this, GT_PHYSREG) GenTreePhysReg(reg, type); - return result; -} - -GenTree* Compiler::gtNewJmpTableNode() -{ - return new (this, GT_JMPTABLE) GenTree(GT_JMPTABLE, TYP_I_IMPL); -} - -/***************************************************************************** - * - * Converts an annotated token into an icon flags (so that we will later be - * able to tell the type of the handle that will be embedded in the icon - * node) - */ - -GenTreeFlags Compiler::gtTokenToIconFlags(unsigned token) -{ - GenTreeFlags flags = GTF_EMPTY; - - switch (TypeFromToken(token)) - { - case mdtTypeRef: - case mdtTypeDef: - case mdtTypeSpec: - flags = GTF_ICON_CLASS_HDL; - break; - - case mdtMethodDef: - flags = GTF_ICON_METHOD_HDL; - break; - - case mdtFieldDef: - flags = GTF_ICON_FIELD_HDL; - break; - - default: - flags = GTF_ICON_TOKEN_HDL; - break; - } - - return flags; -} - -//----------------------------------------------------------------------------------------- -// gtNewIndOfIconHandleNode: Creates an indirection GenTree node of a constant handle -// -// Arguments: -// indType - The type returned by the indirection node -// addr - The constant address to read from -// iconFlags - The GTF_ICON flag value that specifies the kind of handle that we have -// isInvariant - The indNode should also be marked as invariant -// -// Return Value: -// Returns a GT_IND node representing value at the address provided by 'addr' -// -// Notes: -// The GT_IND node is marked as non-faulting. -// If the indirection is not invariant, we also mark the indNode as GTF_GLOB_REF. -// -GenTree* Compiler::gtNewIndOfIconHandleNode(var_types indType, size_t addr, GenTreeFlags iconFlags, bool isInvariant) -{ - GenTree* addrNode = gtNewIconHandleNode(addr, iconFlags); - GenTreeFlags indirFlags = GTF_IND_NONFAULTING; // This indirection won't cause an exception. - - if (isInvariant) - { - // This indirection also is invariant. - indirFlags |= GTF_IND_INVARIANT; - - if (iconFlags == GTF_ICON_STR_HDL) - { - // String literals are never null - indirFlags |= GTF_IND_NONNULL; - } - } - - GenTree* indNode = gtNewIndir(indType, addrNode, indirFlags); - return indNode; -} - -/***************************************************************************** - * - * Allocates a integer constant entry that represents a HANDLE to something. - * It may not be allowed to embed HANDLEs directly into the JITed code (for eg, - * as arguments to JIT helpers). Get a corresponding value that can be embedded. - * If the handle needs to be accessed via an indirection, pValue points to it. - */ - -GenTree* Compiler::gtNewIconEmbHndNode(void* value, void* pValue, GenTreeFlags iconFlags, void* compileTimeHandle) -{ - GenTreeIntCon* iconNode; - GenTree* handleNode; - - if (value != nullptr) - { - // When 'value' is non-null, pValue is required to be null - assert(pValue == nullptr); - - // use 'value' to construct an integer constant node - iconNode = gtNewIconHandleNode((size_t)value, iconFlags); - - // 'value' is the handle - handleNode = iconNode; - } - else - { - // When 'value' is null, pValue is required to be non-null - assert(pValue != nullptr); - - // use 'pValue' to construct an integer constant node - iconNode = gtNewIconHandleNode((size_t)pValue, iconFlags); - - // 'pValue' is an address of a location that contains the handle, construct the indirection of 'pValue'. - handleNode = gtNewIndir(TYP_I_IMPL, iconNode, GTF_IND_NONFAULTING | GTF_IND_INVARIANT); - } - - iconNode->gtCompileTimeHandle = (size_t)compileTimeHandle; -#ifdef DEBUG - if (iconFlags == GTF_ICON_FTN_ADDR) - { - iconNode->gtTargetHandle = (size_t)compileTimeHandle; - } - if (iconFlags == GTF_ICON_OBJ_HDL) - { - iconNode->gtTargetHandle = (size_t)value; - } -#endif - return handleNode; -} - -/*****************************************************************************/ -GenTree* Compiler::gtNewStringLiteralNode(InfoAccessType iat, void* pValue) -{ - GenTree* tree = nullptr; - - switch (iat) - { - case IAT_VALUE: - setMethodHasFrozenObjects(); - tree = gtNewIconEmbHndNode(pValue, nullptr, GTF_ICON_OBJ_HDL, nullptr); -#ifdef DEBUG - tree->AsIntCon()->gtTargetHandle = (size_t)pValue; -#endif - break; - - case IAT_PVALUE: // The value needs to be accessed via an indirection - // Create an indirection - tree = gtNewIndOfIconHandleNode(TYP_REF, (size_t)pValue, GTF_ICON_STR_HDL, true); -#ifdef DEBUG - tree->gtGetOp1()->AsIntCon()->gtTargetHandle = (size_t)pValue; -#endif - break; - - case IAT_PPVALUE: // The value needs to be accessed via a double indirection - // Create the first indirection. - tree = gtNewIndOfIconHandleNode(TYP_I_IMPL, (size_t)pValue, GTF_ICON_CONST_PTR, true); -#ifdef DEBUG - tree->gtGetOp1()->AsIntCon()->gtTargetHandle = (size_t)pValue; -#endif - // Create the second indirection. - tree = gtNewIndir(TYP_REF, tree, GTF_IND_NONFAULTING | GTF_IND_INVARIANT | GTF_IND_NONNULL); - break; - - default: - noway_assert(!"Unexpected InfoAccessType"); - } - - return tree; -} - -//------------------------------------------------------------------------ -// gtNewStringLiteralLength: create GenTreeIntCon node for the given string -// literal to store its length. -// -// Arguments: -// node - string literal node. -// -// Return Value: -// GenTreeIntCon node with string's length as a value or null. -// -GenTreeIntCon* Compiler::gtNewStringLiteralLength(GenTreeStrCon* node) -{ - if (node->IsStringEmptyField()) - { - JITDUMP("Folded String.Empty.Length to 0\n"); - return gtNewIconNode(0); - } - - int length = info.compCompHnd->getStringLiteral(node->gtScpHnd, node->gtSconCPX, nullptr, 0); - if (length >= 0) - { - GenTreeIntCon* iconNode = gtNewIconNode(length); - JITDUMP("Folded 'CNS_STR.Length' to '%d'\n", length) - return iconNode; - } - return nullptr; -} - -/*****************************************************************************/ - -GenTree* Compiler::gtNewLconNode(int64_t value) -{ -#ifdef TARGET_64BIT - GenTree* node = new (this, GT_CNS_INT) GenTreeIntCon(TYP_LONG, value); -#else - GenTree* node = new (this, GT_CNS_LNG) GenTreeLngCon(value); -#endif - - return node; -} - -GenTree* Compiler::gtNewDconNodeF(float value) -{ - return gtNewDconNode(FloatingPointUtils::convertToDouble(value), TYP_FLOAT); -} - -GenTree* Compiler::gtNewDconNodeD(double value) -{ - return gtNewDconNode(value, TYP_DOUBLE); -} - -GenTree* Compiler::gtNewDconNode(double value, var_types type) -{ - return new (this, GT_CNS_DBL) GenTreeDblCon(value, type); -} - -GenTree* Compiler::gtNewSconNode(int CPX, CORINFO_MODULE_HANDLE scpHandle) -{ - // 'GT_CNS_STR' nodes later get transformed into 'GT_CALL' - assert(GenTree::s_gtNodeSizes[GT_CALL] > GenTree::s_gtNodeSizes[GT_CNS_STR]); - GenTree* node = new (this, GT_CALL) GenTreeStrCon(CPX, scpHandle DEBUGARG(/*largeNode*/ true)); - return node; -} - -#if defined(FEATURE_SIMD) -GenTreeVecCon* Compiler::gtNewVconNode(var_types type) -{ - GenTreeVecCon* vecCon = new (this, GT_CNS_VEC) GenTreeVecCon(type); - return vecCon; -} - -GenTreeVecCon* Compiler::gtNewVconNode(var_types type, void* data) -{ - GenTreeVecCon* vecCon = new (this, GT_CNS_VEC) GenTreeVecCon(type); - memcpy(&vecCon->gtSimdVal, data, genTypeSize(type)); - return vecCon; -} -#endif // FEATURE_SIMD - -#if defined(FEATURE_MASKED_HW_INTRINSICS) -GenTreeMskCon* Compiler::gtNewMskConNode(var_types type) -{ - GenTreeMskCon* mskCon = new (this, GT_CNS_MSK) GenTreeMskCon(type); - return mskCon; -} -#endif // FEATURE_MASKED_HW_INTRINSICS - -GenTree* Compiler::gtNewAllBitsSetConNode(var_types type) -{ -#ifdef FEATURE_SIMD - if (varTypeIsSIMD(type)) - { - GenTreeVecCon* allBitsSet = gtNewVconNode(type); - allBitsSet->gtSimdVal = simd_t::AllBitsSet(); - return allBitsSet; - } -#endif // FEATURE_SIMD - - switch (type) - { - case TYP_BYTE: - case TYP_UBYTE: - { - return gtNewIconNode(0xFF); - } - - case TYP_SHORT: - case TYP_USHORT: - { - return gtNewIconNode(0xFFFF); - } - - case TYP_INT: - case TYP_UINT: - { - return gtNewIconNode(-1); - } - - case TYP_LONG: - case TYP_ULONG: - { - return gtNewLconNode(-1); - } - - default: - { - unreached(); - } - } -} - -GenTree* Compiler::gtNewZeroConNode(var_types type) -{ -#ifdef FEATURE_SIMD - if (varTypeIsSIMD(type)) - { - GenTreeVecCon* vecCon = gtNewVconNode(type); - vecCon->gtSimdVal = simd_t::Zero(); - return vecCon; - } -#endif // FEATURE_SIMD - - type = genActualType(type); - switch (type) - { - case TYP_INT: - case TYP_REF: - case TYP_BYREF: - { - return gtNewIconNode(0, type); - } - - case TYP_LONG: - { - return gtNewLconNode(0); - } - - case TYP_FLOAT: - case TYP_DOUBLE: - { - return gtNewDconNode(0.0, type); - } - - default: - { - unreached(); - } - } -} - -GenTree* Compiler::gtNewOneConNode(var_types type, var_types simdBaseType /* = TYP_UNDEF */) -{ -#if defined(FEATURE_SIMD) - if (varTypeIsSIMD(type)) - { - GenTreeVecCon* one = gtNewVconNode(type); - - unsigned simdSize = genTypeSize(type); - uint32_t simdLength = getSIMDVectorLength(simdSize, simdBaseType); - - switch (simdBaseType) - { - case TYP_BYTE: - case TYP_UBYTE: - { - for (uint32_t index = 0; index < simdLength; index++) - { - one->gtSimdVal.u8[index] = 1; - } - break; - } - - case TYP_SHORT: - case TYP_USHORT: - { - for (uint32_t index = 0; index < simdLength; index++) - { - one->gtSimdVal.u16[index] = 1; - } - break; - } - - case TYP_INT: - case TYP_UINT: - { - for (uint32_t index = 0; index < simdLength; index++) - { - one->gtSimdVal.u32[index] = 1; - } - break; - } - - case TYP_LONG: - case TYP_ULONG: - { - for (uint32_t index = 0; index < simdLength; index++) - { - one->gtSimdVal.u64[index] = 1; - } - break; - } - - case TYP_FLOAT: - { - for (uint32_t index = 0; index < simdLength; index++) - { - one->gtSimdVal.f32[index] = 1.0f; - } - break; - } - - case TYP_DOUBLE: - { - for (uint32_t index = 0; index < simdLength; index++) - { - one->gtSimdVal.f64[index] = 1.0; - } - break; - } - - default: - { - unreached(); - } - } - - return one; - } -#endif // FEATURE_SIMD - - switch (type) - { - case TYP_INT: - case TYP_UINT: - { - return gtNewIconNode(1); - } - - case TYP_LONG: - case TYP_ULONG: - { - return gtNewLconNode(1); - } - - case TYP_FLOAT: - case TYP_DOUBLE: - { - return gtNewDconNode(1.0, type); - } - - default: - { - unreached(); - } - } -} - -//------------------------------------------------------------------------ -// gtNewGenericCon: -// Create an IR node representing a constant value of any type. -// -// Parameters: -// type - The primitive type. For small types the constant will be -// zero/sign-extended and a TYP_INT node will be returned. -// cnsVal - Pointer to data -// -// Returns: -// An IR node representing the constant. -// -GenTree* Compiler::gtNewGenericCon(var_types type, uint8_t* cnsVal) -{ - switch (type) - { -#define READ_VALUE(typ) \ - typ val; \ - memcpy(&val, cnsVal, sizeof(typ)); - - case TYP_BYTE: - { - READ_VALUE(int8_t); - return gtNewIconNode(val); - } - case TYP_UBYTE: - { - READ_VALUE(uint8_t); - return gtNewIconNode(val); - } - case TYP_SHORT: - { - READ_VALUE(int16_t); - return gtNewIconNode(val); - } - case TYP_USHORT: - { - READ_VALUE(uint16_t); - return gtNewIconNode(val); - } - case TYP_INT: - { - READ_VALUE(int32_t); - return gtNewIconNode(val); - } - case TYP_LONG: - { - READ_VALUE(int64_t); - return gtNewLconNode(val); - } - case TYP_FLOAT: - { - READ_VALUE(float); - return gtNewDconNodeF(val); - } - case TYP_DOUBLE: - { - READ_VALUE(double); - return gtNewDconNodeD(val); - } - case TYP_REF: - { - READ_VALUE(ssize_t); - if (val == 0) - { - return gtNewNull(); - } - else - { - // Even if the caller doesn't need the resulting tree let's still conservatively call - // setMethodHasFrozenObjects here to make caller's life easier. - setMethodHasFrozenObjects(); - GenTree* tree = gtNewIconEmbHndNode((void*)val, nullptr, GTF_ICON_OBJ_HDL, nullptr); - return tree; - } - } -#ifdef FEATURE_SIMD - case TYP_SIMD8: - case TYP_SIMD12: - case TYP_SIMD16: -#if defined(TARGET_XARCH) - case TYP_SIMD32: - case TYP_SIMD64: -#endif // TARGET_XARCH - { - return gtNewVconNode(type, cnsVal); - } -#endif // FEATURE_SIMD - default: - unreached(); - -#undef READ_VALUE - } -} - -//------------------------------------------------------------------------ -// CreateInitValue: -// Create an IR node representing a constant value with the specified 8 -// byte character broadcast into all of its bytes. -// -// Parameters: -// type - The primitive type. For small types the constant will be -// zero/sign-extended and a TYP_INT node will be returned. -// pattern - A byte pattern. -// -// Returns: -// An IR node representing the constant. -// -// Remarks: -// Should only be called when that pattern can actually be represented; for -// example, GC pointers only support an init pattern of zero. -// -GenTree* Compiler::gtNewConWithPattern(var_types type, uint8_t pattern) -{ - switch (type) - { - case TYP_UBYTE: - return gtNewIconNode(pattern); - case TYP_BYTE: - return gtNewIconNode((int8_t)pattern); - case TYP_SHORT: - return gtNewIconNode((int16_t)(pattern * 0x0101)); - case TYP_USHORT: - return gtNewIconNode((uint16_t)(pattern * 0x0101)); - case TYP_INT: - return gtNewIconNode(pattern * 0x01010101); - case TYP_LONG: - return gtNewLconNode(pattern * 0x0101010101010101LL); - case TYP_FLOAT: - float floatPattern; - memset(&floatPattern, pattern, sizeof(floatPattern)); - return gtNewDconNodeF(floatPattern); - case TYP_DOUBLE: - double doublePattern; - memset(&doublePattern, pattern, sizeof(doublePattern)); - return gtNewDconNodeD(doublePattern); - case TYP_REF: - case TYP_BYREF: - assert(pattern == 0); - return gtNewZeroConNode(type); - -#ifdef FEATURE_SIMD - case TYP_SIMD8: - case TYP_SIMD12: - case TYP_SIMD16: -#if defined(TARGET_XARCH) - case TYP_SIMD32: - case TYP_SIMD64: -#endif // TARGET_XARCH - { - GenTreeVecCon* node = gtNewVconNode(type); - memset(&node->gtSimdVal, pattern, sizeof(node->gtSimdVal)); - return node; - } -#endif // FEATURE_SIMD - - default: - unreached(); - } -} - -//------------------------------------------------------------------------ -// gtNewStoreLclVarNode: Create a local store node. -// -// Arguments: -// lclNum - Number of the local being stored to -// value - Value to store -// -// Return Value: -// The created STORE_LCL_VAR node. -// -GenTreeLclVar* Compiler::gtNewStoreLclVarNode(unsigned lclNum, GenTree* value) -{ - LclVarDsc* varDsc = lvaGetDesc(lclNum); - var_types type = varDsc->lvNormalizeOnLoad() ? varDsc->TypeGet() : genActualType(varDsc); - GenTreeLclVar* store = new (this, GT_STORE_LCL_VAR) GenTreeLclVar(type, lclNum, value); - store->gtFlags |= (GTF_VAR_DEF | GTF_ASG); - if (varDsc->IsAddressExposed()) - { - store->gtFlags |= GTF_GLOB_REF; - } - - gtInitializeStoreNode(store, value); - - return store; -} - -//------------------------------------------------------------------------ -// gtNewStoreLclFldNode: Create a local field store node. -// -// Arguments: -// lclNum - Number of the local being stored to -// type - Type of the store -// layout - Struct layout of the store -// offset - Offset of the store -// value - Value to store -// -// Return Value: -// The created STORE_LCL_FLD node. -// -GenTreeLclFld* Compiler::gtNewStoreLclFldNode( - unsigned lclNum, var_types type, ClassLayout* layout, unsigned offset, GenTree* value) -{ - assert((type == TYP_STRUCT) == (layout != nullptr)); - - GenTreeLclFld* store = new (this, GT_STORE_LCL_FLD) GenTreeLclFld(type, lclNum, offset, value, layout); - store->gtFlags |= (GTF_VAR_DEF | GTF_ASG); - if (store->IsPartialLclFld(this)) - { - store->gtFlags |= GTF_VAR_USEASG; - } - if (lvaGetDesc(lclNum)->IsAddressExposed()) - { - store->gtFlags |= GTF_GLOB_REF; - } - - gtInitializeStoreNode(store, value); - - return store; -} - -GenTreeCall* Compiler::gtNewIndCallNode(GenTree* addr, var_types type, const DebugInfo& di) -{ - return gtNewCallNode(CT_INDIRECT, (CORINFO_METHOD_HANDLE)addr, type, di); -} - -GenTreeCall* Compiler::gtNewCallNode(gtCallTypes callType, - CORINFO_METHOD_HANDLE callHnd, - var_types type, - const DebugInfo& di) -{ - GenTreeCall* node = new (this, GT_CALL) GenTreeCall(genActualType(type)); - - node->gtFlags |= (GTF_CALL | GTF_GLOB_REF); -#ifdef UNIX_X86_ABI - if (callType == CT_INDIRECT || callType == CT_HELPER) - node->gtFlags |= GTF_CALL_POP_ARGS; -#endif // UNIX_X86_ABI - node->gtCallType = callType; - node->gtCallMethHnd = callHnd; - INDEBUG(node->callSig = nullptr;) - node->tailCallInfo = nullptr; - node->gtRetClsHnd = nullptr; - node->gtControlExpr = nullptr; - node->gtCallMoreFlags = GTF_CALL_M_EMPTY; - INDEBUG(node->gtCallDebugFlags = GTF_CALL_MD_EMPTY); - node->gtInlineInfoCount = 0; - - if (callType == CT_INDIRECT) - { - node->gtCallCookie = nullptr; - } - else - { - node->ClearInlineInfo(); - } - node->gtReturnType = type; - -#ifdef FEATURE_READYTORUN - node->gtEntryPoint.addr = nullptr; - node->gtEntryPoint.accessType = IAT_VALUE; -#endif - -#if defined(DEBUG) - // These get updated after call node is built. - node->gtInlineObservation = InlineObservation::CALLEE_UNUSED_INITIAL; - node->gtRawILOffset = BAD_IL_OFFSET; - node->gtInlineContext = compInlineContext; -#endif - - // Spec: Managed Retval sequence points needs to be generated while generating debug info for debuggable code. - // - // Implementation note: if not generating MRV info genCallSite2ILOffsetMap will be NULL and - // codegen will pass DebugInfo() to emitter, which will cause emitter - // not to emit IP mapping entry. - if (opts.compDbgCode && opts.compDbgInfo && di.IsValid()) - { - // Managed Retval - IL offset of the call. This offset is used to emit a - // CALL_INSTRUCTION type sequence point while emitting corresponding native call. - // - // TODO-Cleanup: - // a) (Opt) We need not store this offset if the method doesn't return a - // value. Rather it can be made BAD_IL_OFFSET to prevent a sequence - // point being emitted. - // - // b) (Opt) Add new sequence points only if requested by debugger through - // a new boundary type - ICorDebugInfo::BoundaryTypes - if (genCallSite2DebugInfoMap == nullptr) - { - genCallSite2DebugInfoMap = new (getAllocator()) CallSiteDebugInfoTable(getAllocator()); - } - - // Make sure that there are no duplicate entries for a given call node - assert(!genCallSite2DebugInfoMap->Lookup(node)); - genCallSite2DebugInfoMap->Set(node, di); - } - - // Initialize gtOtherRegs - node->ClearOtherRegs(); - - // Initialize spill flags of gtOtherRegs - node->ClearOtherRegFlags(); - -#if !defined(TARGET_64BIT) - if (varTypeIsLong(node)) - { - assert(node->gtReturnType == node->gtType); - // Initialize Return type descriptor of call node - node->InitializeLongReturnType(); - } -#endif // !defined(TARGET_64BIT) - - return node; -} - -GenTreeLclVar* Compiler::gtNewLclvNode(unsigned lnum, var_types type DEBUGARG(IL_OFFSET offs)) -{ - assert(type != TYP_VOID); - // We need to ensure that all struct values are normalized. - // It might be nice to assert this in general, but we have stores of int to long. - if (varTypeIsStruct(type)) - { - // Make an exception for implicit by-ref parameters during global morph, since - // their lvType has been updated to byref but their appearances have not yet all - // been rewritten and so may have struct type still. - LclVarDsc* varDsc = lvaGetDesc(lnum); - - bool simd12ToSimd16Widening = false; -#if defined(FEATURE_SIMD) - // We can additionally have a SIMD12 that was widened to a SIMD16, generally as part of lowering - simd12ToSimd16Widening = (type == TYP_SIMD16) && (varDsc->lvType == TYP_SIMD12); -#endif // FEATURE_SIMD - assert((type == varDsc->lvType) || simd12ToSimd16Widening || - (lvaIsImplicitByRefLocal(lnum) && fgGlobalMorph && (varDsc->lvType == TYP_BYREF))); - } - GenTreeLclVar* node = new (this, GT_LCL_VAR) GenTreeLclVar(GT_LCL_VAR, type, lnum DEBUGARG(offs)); - - /* Cannot have this assert because the inliner uses this function - * to add temporaries */ - - // assert(lnum < lvaCount); - - return node; -} - -GenTreeLclVar* Compiler::gtNewLclVarNode(unsigned lclNum, var_types type) -{ - LclVarDsc* varDsc = lvaGetDesc(lclNum); - if (type == TYP_UNDEF) - { - type = varDsc->lvNormalizeOnLoad() ? varDsc->TypeGet() : genActualType(varDsc); - } - - GenTreeLclVar* lclVar = gtNewLclvNode(lclNum, type); - if (varDsc->IsAddressExposed()) - { - lclVar->gtFlags |= GTF_GLOB_REF; - } - - return lclVar; -} - -GenTreeLclVar* Compiler::gtNewLclLNode(unsigned lnum, var_types type DEBUGARG(IL_OFFSET offs)) -{ - // We need to ensure that all struct values are normalized. - // It might be nice to assert this in general, but we have stores of int to long. - if (varTypeIsStruct(type)) - { - // Make an exception for implicit by-ref parameters during global morph, since - // their lvType has been updated to byref but their appearances have not yet all - // been rewritten and so may have struct type still. - assert(type == lvaTable[lnum].lvType || - (lvaIsImplicitByRefLocal(lnum) && fgGlobalMorph && (lvaTable[lnum].lvType == TYP_BYREF))); - } - // This local variable node may later get transformed into a large node - assert(GenTree::s_gtNodeSizes[LargeOpOpcode()] > GenTree::s_gtNodeSizes[GT_LCL_VAR]); - GenTreeLclVar* node = - new (this, LargeOpOpcode()) GenTreeLclVar(GT_LCL_VAR, type, lnum DEBUGARG(offs) DEBUGARG(/*largeNode*/ true)); - return node; -} - -GenTreeLclFld* Compiler::gtNewLclVarAddrNode(unsigned lclNum, var_types type) -{ - GenTreeLclFld* node = gtNewLclAddrNode(lclNum, 0, type); - return node; -} - -GenTreeLclFld* Compiler::gtNewLclAddrNode(unsigned lclNum, unsigned lclOffs, var_types type) -{ - GenTreeLclFld* node = new (this, GT_LCL_ADDR) GenTreeLclFld(GT_LCL_ADDR, type, lclNum, lclOffs); - return node; -} - -GenTreeConditional* Compiler::gtNewConditionalNode( - genTreeOps oper, GenTree* cond, GenTree* op1, GenTree* op2, var_types type) -{ - assert(GenTree::OperIsConditional(oper)); - GenTreeConditional* node = new (this, oper) GenTreeConditional(oper, type, cond, op1, op2); - node->gtFlags |= (cond->gtFlags & GTF_ALL_EFFECT); - node->gtFlags |= (op1->gtFlags & GTF_ALL_EFFECT); - node->gtFlags |= (op2->gtFlags & GTF_ALL_EFFECT); - return node; -} - -GenTreeLclFld* Compiler::gtNewLclFldNode(unsigned lnum, var_types type, unsigned offset, ClassLayout* layout) -{ - GenTreeLclFld* node = new (this, GT_LCL_FLD) GenTreeLclFld(GT_LCL_FLD, type, lnum, offset, layout); - return node; -} - -GenTreeRetExpr* Compiler::gtNewInlineCandidateReturnExpr(GenTreeCall* inlineCandidate, var_types type) -{ - assert(GenTree::s_gtNodeSizes[GT_RET_EXPR] == TREE_NODE_SZ_LARGE); - - GenTreeRetExpr* node = new (this, GT_RET_EXPR) GenTreeRetExpr(type); - - node->gtInlineCandidate = inlineCandidate; - - node->gtSubstExpr = nullptr; - node->gtSubstBB = nullptr; - - // GT_RET_EXPR node eventually might be turned back into GT_CALL (when inlining is aborted for example). - // Therefore it should carry the GTF_CALL flag so that all the rules about spilling can apply to it as well. - // For example, impImportLeave or CEE_POP need to spill GT_RET_EXPR before empty the evaluation stack. - node->gtFlags |= GTF_CALL; - - return node; -} - -//------------------------------------------------------------------------ -// gtNewFieldAddrNode: Create a new GT_FIELD_ADDR node. -// -// TODO-ADDR: consider creating a variant of this which would skip various -// no-op constructs (such as struct fields with zero offsets), and fold -// others (LCL_VAR_ADDR + FIELD_ADDR => LCL_FLD_ADDR). -// -// Arguments: -// type - type for the address node -// fldHnd - the field handle -// obj - the instance, an address -// offset - the field offset -// -// Return Value: -// The created node. -// -GenTreeFieldAddr* Compiler::gtNewFieldAddrNode(var_types type, CORINFO_FIELD_HANDLE fldHnd, GenTree* obj, DWORD offset) -{ - assert(varTypeIsI(genActualType(type))); - - GenTreeFieldAddr* fieldNode = new (this, GT_FIELD_ADDR) GenTreeFieldAddr(type, obj, fldHnd, offset); - - // If "obj" is the address of a local, note that a field of that struct local has been accessed. - if ((obj != nullptr) && obj->IsLclVarAddr()) - { - LclVarDsc* varDsc = lvaGetDesc(obj->AsLclVarCommon()); - varDsc->lvFieldAccessed = 1; - } - - if ((obj != nullptr) && fgAddrCouldBeNull(obj)) - { - fieldNode->gtFlags |= GTF_EXCEPT; - } - - return fieldNode; -} - -//------------------------------------------------------------------------ -// gtInitializeStoreNode: Initialize a store node. -// -// Common initialization for all STORE nodes. Marks SIMD locals as "used in -// a HW intrinsic". -// -// Arguments: -// store - The store node -// data - The value to store -// -void Compiler::gtInitializeStoreNode(GenTree* store, GenTree* value) -{ - // TODO-ASG: add asserts that the types match here. - assert(store->Data() == value); - -#if defined(FEATURE_SIMD) -#ifndef TARGET_X86 - if (varTypeIsSIMD(store)) - { - // TODO-ASG: delete this zero-diff quirk. - if (!value->IsCall() || !value->AsCall()->ShouldHaveRetBufArg()) - { - // We want to track SIMD stores as being intrinsics since they are - // functionally SIMD `mov` instructions and are more efficient when - // we don't promote, particularly when it occurs due to inlining. - SetOpLclRelatedToSIMDIntrinsic(store); - SetOpLclRelatedToSIMDIntrinsic(value); - } - } -#else // TARGET_X86 - // TODO-Cleanup: merge into the all-arch. - if (varTypeIsSIMD(value) || varTypeIsMask(value)) - { - bool isRelatedToSimdIntrinsic = value->OperIs(GT_HWINTRINSIC, GT_CNS_VEC); - -#if defined(FEATURE_MASKED_HW_INTRINSICS) - isRelatedToSimdIntrinsic |= value->OperIs(GT_CNS_MSK); -#endif // FEATURE_MASKED_HW_INTRINSICS - - if (isRelatedToSimdIntrinsic) - { - SetOpLclRelatedToSIMDIntrinsic(store); - } - } -#endif // TARGET_X86 -#endif // FEATURE_SIMD -} - -//------------------------------------------------------------------------ -// gtInitializeIndirNode: Initialize an indirection node. -// -// Sets side effect flags based on the passed-in indirection flags. -// -// Arguments: -// indir - The indirection node -// indirFlags - The indirection flags -// -void Compiler::gtInitializeIndirNode(GenTreeIndir* indir, GenTreeFlags indirFlags) -{ - assert(varTypeIsI(genActualType(indir->Addr()))); - assert((indirFlags & ~GTF_IND_FLAGS) == GTF_EMPTY); - - indir->gtFlags |= indirFlags; - indir->SetIndirExceptionFlags(this); - - if ((indirFlags & GTF_IND_INVARIANT) == 0) - { - indir->gtFlags |= GTF_GLOB_REF; - } - if ((indirFlags & GTF_IND_VOLATILE) != 0) - { - indir->SetHasOrderingSideEffect(); - } -} - -//------------------------------------------------------------------------ -// gtNewBlkIndir: Create a struct indirection node. -// -// Arguments: -// layout - The struct layout -// addr - Address of the indirection -// indirFlags - Indirection flags -// -// Return Value: -// The created GT_BLK node. -// -GenTreeBlk* Compiler::gtNewBlkIndir(ClassLayout* layout, GenTree* addr, GenTreeFlags indirFlags) -{ - assert(layout->GetType() == TYP_STRUCT); - GenTreeBlk* blkNode = new (this, GT_BLK) GenTreeBlk(GT_BLK, TYP_STRUCT, addr, layout); - gtInitializeIndirNode(blkNode, indirFlags); - - return blkNode; -} - -//------------------------------------------------------------------------ -// gtNewMemoryBarrier: Create a memory barrier node -// -// Arguments: -// barrierKind - the kind of barrer we are creating -// -// Return Value: -// The created GT_MEMORYBARRIER node. -// -GenTree* Compiler::gtNewMemoryBarrier(BarrierKind barrierKind) -{ - GenTree* tree = new (this, GT_MEMORYBARRIER) GenTree(GT_MEMORYBARRIER, TYP_VOID); - tree->gtFlags |= GTF_GLOB_REF | GTF_ASG; - if (barrierKind == BARRIER_LOAD_ONLY) - { - tree->gtFlags |= GTF_MEMORYBARRIER_LOAD; - } - else if (barrierKind == BARRIER_STORE_ONLY) - { - tree->gtFlags |= GTF_MEMORYBARRIER_STORE; - } - return tree; -} - -//------------------------------------------------------------------------------ -// gtNewIndir : Create an indirection node. -// -// Arguments: -// typ - Type of the node -// addr - Address of the indirection -// indirFlags - Indirection flags -// -// Return Value: -// The created GT_IND node. -// -GenTreeIndir* Compiler::gtNewIndir(var_types typ, GenTree* addr, GenTreeFlags indirFlags) -{ - GenTreeIndir* indir = new (this, GT_IND) GenTreeIndir(GT_IND, typ, addr, nullptr); - gtInitializeIndirNode(indir, indirFlags); - - return indir; -} - -//------------------------------------------------------------------------ -// gtNewLoadValueNode: Return a node that represents a loaded value. -// -// Arguments: -// type - Type to load -// layout - Struct layout to load -// addr - The address -// indirFlags - Indirection flags -// -// Return Value: -// A "BLK/IND" node, or "LCL_VAR" if "addr" points to a compatible local. -// -GenTree* Compiler::gtNewLoadValueNode(var_types type, ClassLayout* layout, GenTree* addr, GenTreeFlags indirFlags) -{ - assert(((indirFlags & ~GTF_IND_FLAGS) == 0) && ((type != TYP_STRUCT) || (layout != nullptr))); - - if (((indirFlags & GTF_IND_VOLATILE) == 0) && addr->IsLclVarAddr()) - { - unsigned lclNum = addr->AsLclFld()->GetLclNum(); - LclVarDsc* varDsc = lvaGetDesc(lclNum); - if ((varDsc->TypeGet() == type) && - ((type != TYP_STRUCT) || ClassLayout::AreCompatible(layout, varDsc->GetLayout()))) - { - return gtNewLclvNode(lclNum, type); - } - } - - return (type == TYP_STRUCT) ? gtNewBlkIndir(layout, addr, indirFlags) : gtNewIndir(type, addr, indirFlags); -} - -//------------------------------------------------------------------------------ -// gtNewStoreBlkNode: Create an indirect struct store node. -// -// Arguments: -// layout - The struct layout -// addr - Destination address -// value - Value to store -// indirFlags - Indirection flags -// -// Return Value: -// The created GT_STORE_BLK node. -// -GenTreeBlk* Compiler::gtNewStoreBlkNode(ClassLayout* layout, GenTree* addr, GenTree* value, GenTreeFlags indirFlags) -{ - assert((indirFlags & GTF_IND_INVARIANT) == 0); - assert(value->IsInitVal() || ClassLayout::AreCompatible(layout, value->GetLayout(this))); - - GenTreeBlk* store = new (this, GT_STORE_BLK) GenTreeBlk(GT_STORE_BLK, TYP_STRUCT, addr, value, layout); - store->gtFlags |= GTF_ASG; - gtInitializeIndirNode(store, indirFlags); - gtInitializeStoreNode(store, value); - - return store; -} - -//------------------------------------------------------------------------------ -// gtNewStoreIndNode : Create an indirect store node. -// -// Arguments: -// type - Type of the store -// addr - Destination address -// value - Value to store -// indirFlags - Indirection flags -// -// Return Value: -// The created GT_STOREIND node. -// -GenTreeStoreInd* Compiler::gtNewStoreIndNode(var_types type, GenTree* addr, GenTree* value, GenTreeFlags indirFlags) -{ - assert(((indirFlags & GTF_IND_INVARIANT) == 0) && (type != TYP_STRUCT)); - - GenTreeStoreInd* store = new (this, GT_STOREIND) GenTreeStoreInd(type, addr, value); - store->gtFlags |= GTF_ASG; - gtInitializeIndirNode(store, indirFlags); - gtInitializeStoreNode(store, value); - - return store; -} - -//------------------------------------------------------------------------ -// gtNewStoreValueNode: Return a node that represents a store. -// -// Arguments: -// type - Type to store -// layout - Struct layout for the store -// addr - Destination address -// value - Value to store -// indirFlags - Indirection flags -// -// Return Value: -// A "STORE_BLK/STORE_IND" node, or "STORE_LCL_VAR" if "addr" points to -// a compatible local. -// -GenTree* Compiler::gtNewStoreValueNode( - var_types type, ClassLayout* layout, GenTree* addr, GenTree* value, GenTreeFlags indirFlags) -{ - assert((type != TYP_STRUCT) || (layout != nullptr)); - - if (((indirFlags & GTF_IND_VOLATILE) == 0) && addr->IsLclVarAddr()) - { - unsigned lclNum = addr->AsLclFld()->GetLclNum(); - LclVarDsc* varDsc = lvaGetDesc(lclNum); - if ((varDsc->TypeGet() == type) && - ((type != TYP_STRUCT) || ClassLayout::AreCompatible(layout, varDsc->GetLayout()))) - { - return gtNewStoreLclVarNode(lclNum, value); - } - } - - GenTree* store; - if (type == TYP_STRUCT) - { - store = gtNewStoreBlkNode(layout, addr, value, indirFlags); - } - else - { - store = gtNewStoreIndNode(type, addr, value, indirFlags); - } - - return store; -} - -//------------------------------------------------------------------------ -// gtNewAtomicNode: Create a new atomic operation node. -// -// Arguments: -// oper - The atomic oper -// type - Type to store/load -// addr - Destination ("location") address -// value - Value -// comparand - Comparand value for a CMPXCHG -// -// Return Value: -// The created node. -// -GenTree* Compiler::gtNewAtomicNode(genTreeOps oper, var_types type, GenTree* addr, GenTree* value, GenTree* comparand) -{ - assert(GenTree::OperIsAtomicOp(oper) && ((oper == GT_CMPXCHG) == (comparand != nullptr))); - GenTreeIndir* node; - if (comparand != nullptr) - { - node = new (this, GT_CMPXCHG) GenTreeCmpXchg(type, addr, value, comparand); - addr->gtFlags |= GTF_DONT_CSE; - } - else - { - node = new (this, oper) GenTreeIndir(oper, type, addr, value); - } - - node->AddAllEffectsFlags(GTF_ASG); // All atomics are opaque global stores. - gtInitializeIndirNode(node, GTF_EMPTY); - - return node; -} - -//------------------------------------------------------------------------ -// FixupInitBlkValue: Fixup the init value for an initBlk operation -// -// Arguments: -// type - The type of store that the initBlk is being transformed into -// -// Return Value: -// Modifies the constant value on this node to be the appropriate "fill" -// value for the initblk. -// -// Notes: -// The initBlk MSIL instruction takes a byte value, which must be extended -// to the size of the store when an initBlk is transformed to a store of -// a primitive type. This performs the appropriate extension. -// -void GenTreeIntCon::FixupInitBlkValue(var_types type) -{ - assert(varTypeIsIntegralOrI(type)); - unsigned size = genTypeSize(type); - if (size > 1) - { - size_t cns = gtIconVal; - cns = cns & 0xFF; - cns |= cns << 8; - if (size >= 4) - { - cns |= cns << 16; -#ifdef TARGET_64BIT - if (size == 8) - { - cns |= cns << 32; - } -#endif // TARGET_64BIT - - // Make the type match for evaluation types. - gtType = type; - - // if we are initializing a GC type the value being assigned must be zero (null). - assert(!varTypeIsGC(type) || (cns == 0)); - } - - gtIconVal = cns; - } -} - -//---------------------------------------------------------------------------- -// UsesDivideByConstOptimized: -// returns true if rationalize will use the division by constant -// optimization for this node. -// -// Arguments: -// this - a GenTreeOp node -// comp - the compiler instance -// -// Return Value: -// Return true iff the node is a GT_DIV,GT_UDIV, GT_MOD or GT_UMOD with -// an integer constant and we can perform the division operation using -// a reciprocal multiply or a shift operation. -// -bool GenTreeOp::UsesDivideByConstOptimized(Compiler* comp) -{ - if (!comp->opts.OptimizationEnabled()) - { - return false; - } - - if (!OperIs(GT_DIV, GT_MOD, GT_UDIV, GT_UMOD)) - { - return false; - } -#if defined(TARGET_ARM64) - if (OperIs(GT_MOD, GT_UMOD)) - { - // MOD, UMOD not supported for ARM64 - return false; - } -#endif // TARGET_ARM64 - - bool isSignedDivide = OperIs(GT_DIV, GT_MOD); - GenTree* dividend = gtGetOp1()->gtEffectiveVal(); - GenTree* divisor = gtGetOp2()->gtEffectiveVal(); - -#if !defined(TARGET_64BIT) - if (dividend->OperIs(GT_LONG)) - { - return false; - } -#endif - - if (dividend->IsCnsIntOrI()) - { - // We shouldn't see a divmod with constant operands here but if we do then it's likely - // because optimizations are disabled or it's a case that's supposed to throw an exception. - // Don't optimize this. - return false; - } - - ssize_t divisorValue; - if (divisor->IsCnsIntOrI()) - { - divisorValue = static_cast(divisor->AsIntCon()->IconValue()); - } - else - { - ValueNum vn = divisor->gtVNPair.GetLiberal(); - if (comp->vnStore && comp->vnStore->IsVNConstant(vn)) - { - divisorValue = comp->vnStore->CoercedConstantValue(vn); - } - else - { - return false; - } - } - - const var_types divType = TypeGet(); - - if (divisorValue == 0) - { - // x / 0 and x % 0 can't be optimized because they are required to throw an exception. - return false; - } - else if (isSignedDivide) - { - if (divisorValue == -1) - { - // x / -1 can't be optimized because INT_MIN / -1 is required to throw an exception. - return false; - } - else if (isPow2(divisorValue)) - { - return true; - } - } - else // unsigned divide - { - if (divType == TYP_INT) - { - // Clear up the upper 32 bits of the value, they may be set to 1 because constants - // are treated as signed and stored in ssize_t which is 64 bit in size on 64 bit targets. - divisorValue &= UINT32_MAX; - } - - size_t unsignedDivisorValue = (size_t)divisorValue; - if (isPow2(unsignedDivisorValue)) - { - return true; - } - } - - const bool isDiv = OperIs(GT_DIV, GT_UDIV); - - if (isDiv) - { - if (isSignedDivide) - { - // If the divisor is the minimum representable integer value then the result is either 0 or 1 - if ((divType == TYP_INT && divisorValue == INT_MIN) || (divType == TYP_LONG && divisorValue == INT64_MIN)) - { - return true; - } - } - else - { - // If the divisor is greater or equal than 2^(N - 1) then the result is either 0 or 1 - if (((divType == TYP_INT) && ((UINT32)divisorValue > (UINT32_MAX / 2))) || - ((divType == TYP_LONG) && ((UINT64)divisorValue > (UINT64_MAX / 2)))) - { - return true; - } - } - } - -// TODO-ARM-CQ: Currently there's no GT_MULHI for ARM32 -#if defined(TARGET_XARCH) || defined(TARGET_ARM64) || defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64) - if (!comp->opts.MinOpts() && ((divisorValue >= 3) || !isSignedDivide)) - { - // All checks pass we can perform the division operation using a reciprocal multiply. - return true; - } -#endif - - return false; -} - -//------------------------------------------------------------------------ -// CheckDivideByConstOptimized: -// Checks if we can use the division by constant optimization -// on this node -// and if so sets the flag GTF_DIV_BY_CNS_OPT and -// set GTF_DONT_CSE on the constant node -// -// Arguments: -// this - a GenTreeOp node -// comp - the compiler instance -// -void GenTreeOp::CheckDivideByConstOptimized(Compiler* comp) -{ - if (UsesDivideByConstOptimized(comp)) - { - // Now set DONT_CSE on the GT_CNS_INT divisor, note that - // with ValueNumbering we can have a non GT_CNS_INT divisor - GenTree* divisor = gtGetOp2()->gtEffectiveVal(); - if (divisor->OperIs(GT_CNS_INT)) - { - divisor->gtFlags |= GTF_DONT_CSE; - } - } -} - -//------------------------------------------------------------------------ -// gtNewPutArgReg: Creates a new PutArgReg node. -// -// Arguments: -// type - The actual type of the argument -// arg - The argument node -// argReg - The register that the argument will be passed in -// -// Return Value: -// Returns the newly created PutArgReg node. -// -GenTree* Compiler::gtNewPutArgReg(var_types type, GenTree* arg, regNumber argReg) -{ - assert(arg != nullptr); - - GenTree* node = gtNewOperNode(GT_PUTARG_REG, type, arg); - node->SetRegNum(argReg); - - return node; -} - -//------------------------------------------------------------------------ -// gtNewBitCastNode: Creates a new BitCast node. -// -// Arguments: -// type - The actual type of the argument -// arg - The argument node -// argReg - The register that the argument will be passed in -// -// Return Value: -// Returns the newly created BitCast node. -// -// Notes: -// The node is generated as GenTreeMultiRegOp on RyuJIT/arm, as GenTreeOp on all the other archs. -// -GenTree* Compiler::gtNewBitCastNode(var_types type, GenTree* arg) -{ - assert(arg != nullptr); - assert(type != TYP_STRUCT); - - GenTree* node = nullptr; -#if defined(TARGET_ARM) - // A BITCAST could be a MultiRegOp on arm since we could move a double register to two int registers. - node = new (this, GT_BITCAST) GenTreeMultiRegOp(GT_BITCAST, type, arg, nullptr); -#else - node = gtNewOperNode(GT_BITCAST, type, arg); -#endif - - return node; -} - -//------------------------------------------------------------------------ -// gtNewAllocObjNode: Helper to create an object allocation node. -// -// Arguments: -// pResolvedToken - Resolved token for the object being allocated -// useParent - true iff the token represents a child of the object's class -// -// Return Value: -// Returns GT_ALLOCOBJ node that will be later morphed into an -// allocation helper call or local variable allocation on the stack. -// -// Node creation can fail for inlinees when the type described by pResolvedToken -// can't be represented in jitted code. If this happens, this method will return -// nullptr. -// -GenTreeAllocObj* Compiler::gtNewAllocObjNode(CORINFO_RESOLVED_TOKEN* pResolvedToken, - CORINFO_METHOD_HANDLE callerHandle, - bool useParent) -{ - const bool mustRestoreHandle = true; - bool* const pRuntimeLookup = nullptr; - bool usingReadyToRunHelper = false; - CorInfoHelpFunc helper = CORINFO_HELP_UNDEF; - GenTree* opHandle = impTokenToHandle(pResolvedToken, pRuntimeLookup, mustRestoreHandle, useParent); - -#ifdef FEATURE_READYTORUN - CORINFO_CONST_LOOKUP lookup = {}; - - if (opts.IsReadyToRun()) - { - helper = CORINFO_HELP_READYTORUN_NEW; - CORINFO_LOOKUP_KIND* const pGenericLookupKind = nullptr; - usingReadyToRunHelper = - info.compCompHnd->getReadyToRunHelper(pResolvedToken, pGenericLookupKind, helper, callerHandle, &lookup); - } -#endif - - if (!usingReadyToRunHelper) - { - if (opHandle == nullptr) - { - // We must be backing out of an inline. - assert(compDonotInline()); - return nullptr; - } - } - - bool helperHasSideEffects; - CorInfoHelpFunc helperTemp = info.compCompHnd->getNewHelper(pResolvedToken->hClass, &helperHasSideEffects); - - if (!usingReadyToRunHelper) - { - helper = helperTemp; - } - - // TODO: ReadyToRun: When generic dictionary lookups are necessary, replace the lookup call - // and the newfast call with a single call to a dynamic R2R cell that will: - // 1) Load the context - // 2) Perform the generic dictionary lookup and caching, and generate the appropriate stub - // 3) Allocate and return the new object for boxing - // Reason: performance (today, we'll always use the slow helper for the R2R generics case) - - GenTreeAllocObj* allocObj = - gtNewAllocObjNode(helper, helperHasSideEffects, pResolvedToken->hClass, TYP_REF, opHandle); - -#ifdef FEATURE_READYTORUN - if (usingReadyToRunHelper) - { - assert(lookup.addr != nullptr); - allocObj->gtEntryPoint = lookup; - } -#endif - - return allocObj; -} - -/***************************************************************************** - * - * Clones the given tree value and returns a copy of the given tree. - * If 'complexOK' is false, the cloning is only done provided the tree - * is not too complex (whatever that may mean); - * If 'complexOK' is true, we try slightly harder to clone the tree. - * In either case, NULL is returned if the tree cannot be cloned - * - * Note that there is the function gtCloneExpr() which does a more - * complete job if you can't handle this function failing. - */ - -GenTree* Compiler::gtClone(GenTree* tree, bool complexOK) -{ - GenTree* copy; - - switch (tree->gtOper) - { - case GT_CNS_INT: - -#if defined(LATE_DISASM) - if (tree->IsIconHandle()) - { - copy = gtNewIconHandleNode(tree->AsIntCon()->gtIconVal, tree->gtFlags, tree->AsIntCon()->gtFieldSeq); - copy->AsIntCon()->gtCompileTimeHandle = tree->AsIntCon()->gtCompileTimeHandle; - copy->gtType = tree->gtType; - } - else -#endif - { - copy = new (this, GT_CNS_INT) - GenTreeIntCon(tree->gtType, tree->AsIntCon()->gtIconVal, tree->AsIntCon()->gtFieldSeq); - copy->AsIntCon()->gtCompileTimeHandle = tree->AsIntCon()->gtCompileTimeHandle; - } - break; - - case GT_CNS_LNG: - copy = gtNewLconNode(tree->AsLngCon()->gtLconVal); - break; - - case GT_CNS_DBL: - { - copy = gtNewDconNode(tree->AsDblCon()->DconValue(), tree->TypeGet()); - break; - } - -#if defined(FEATURE_SIMD) - case GT_CNS_VEC: - { - GenTreeVecCon* vecCon = gtNewVconNode(tree->TypeGet()); - vecCon->gtSimdVal = tree->AsVecCon()->gtSimdVal; - copy = vecCon; - break; - } -#endif // FEATURE_SIMD - -#if defined(FEATURE_MASKED_HW_INTRINSICS) - case GT_CNS_MSK: - { - GenTreeMskCon* mskCon = gtNewMskConNode(tree->TypeGet()); - mskCon->gtSimdMaskVal = tree->AsMskCon()->gtSimdMaskVal; - copy = mskCon; - break; - } -#endif // FEATURE_MASKED_HW_INTRINSICS - - case GT_FTN_ADDR: - { - copy = new (this, tree->OperGet()) GenTreeFptrVal(tree->TypeGet(), tree->AsFptrVal()->gtFptrMethod); - copy->AsFptrVal()->gtFptrDelegateTarget = tree->AsFptrVal()->gtFptrDelegateTarget; - -#ifdef FEATURE_READYTORUN - copy->AsFptrVal()->gtEntryPoint = tree->AsFptrVal()->gtEntryPoint; -#endif - break; - } - - case GT_LCL_VAR: - copy = new (this, tree->OperGet()) - GenTreeLclVar(tree->OperGet(), tree->TypeGet(), tree->AsLclVar()->GetLclNum()); - goto FINISH_CLONING_LCL_NODE; - - case GT_LCL_ADDR: - if (!complexOK && (tree->AsLclFld()->GetLclOffs() == 0)) - { - return nullptr; - } - FALLTHROUGH; - - case GT_LCL_FLD: - copy = new (this, tree->OperGet()) - GenTreeLclFld(tree->OperGet(), tree->TypeGet(), tree->AsLclFld()->GetLclNum(), - tree->AsLclFld()->GetLclOffs(), tree->AsLclFld()->GetLayout()); - goto FINISH_CLONING_LCL_NODE; - - FINISH_CLONING_LCL_NODE: - // Remember that the local node has been cloned. Below the flag will be set on 'copy' too. - tree->gtFlags |= GTF_VAR_MOREUSES; - copy->AsLclVarCommon()->SetSsaNum(tree->AsLclVarCommon()->GetSsaNum()); - assert(!copy->AsLclVarCommon()->HasSsaName() || ((copy->gtFlags & GTF_VAR_DEF) == 0)); - break; - - default: - if (!complexOK) - { - return nullptr; - } - - if (tree->OperIs(GT_IND, GT_BLK) && tree->AsIndir()->Addr()->OperIs(GT_FIELD_ADDR)) - { - GenTree* objp = nullptr; - GenTreeFieldAddr* addr = tree->AsIndir()->Addr()->AsFieldAddr(); - - if (addr->IsInstance()) - { - objp = gtClone(addr->GetFldObj(), false); - if (objp == nullptr) - { - return nullptr; - } - } - - copy = gtNewFieldAddrNode(addr->TypeGet(), addr->gtFldHnd, objp, addr->gtFldOffset); - copy->AsFieldAddr()->gtFldMayOverlap = addr->gtFldMayOverlap; - copy->AsFieldAddr()->SetIsSpanLength(addr->IsSpanLength()); -#ifdef FEATURE_READYTORUN - copy->AsFieldAddr()->gtFieldLookup = addr->gtFieldLookup; -#endif - copy = tree->OperIs(GT_BLK) ? gtNewBlkIndir(tree->AsBlk()->GetLayout(), copy) - : gtNewIndir(tree->TypeGet(), copy); - impAnnotateFieldIndir(copy->AsIndir()); - } - else if (tree->OperIs(GT_ADD, GT_SUB)) - { - GenTree* op1 = tree->AsOp()->gtOp1; - GenTree* op2 = tree->AsOp()->gtOp2; - - if (op1->OperIsLeaf() && op2->OperIsLeaf()) - { - op1 = gtClone(op1); - if (op1 == nullptr) - { - return nullptr; - } - op2 = gtClone(op2); - if (op2 == nullptr) - { - return nullptr; - } - - copy = gtNewOperNode(tree->OperGet(), tree->TypeGet(), op1, op2); - } - else - { - return nullptr; - } - } - else - { - return nullptr; - } - - break; - } - - copy->gtFlags |= tree->gtFlags & ~GTF_NODE_MASK; -#if defined(DEBUG) - copy->gtDebugFlags |= tree->gtDebugFlags & ~GTF_DEBUG_NODE_MASK; -#endif // defined(DEBUG) - - return copy; -} - -//------------------------------------------------------------------------ -// gtCloneExpr: Create a copy of `tree` -// -// Arguments: -// tree - GenTree to create a copy of -// -// Return Value: -// A copy of the given tree. - -GenTree* Compiler::gtCloneExpr(GenTree* tree) -{ - if (tree == nullptr) - { - return nullptr; - } - - /* Figure out what kind of a node we have */ - - genTreeOps oper = tree->OperGet(); - unsigned kind = tree->OperKind(); - GenTree* copy; - - /* Is this a leaf node? */ - - if (kind & GTK_LEAF) - { - switch (oper) - { - case GT_CNS_INT: - -#if defined(LATE_DISASM) - if (tree->IsIconHandle()) - { - copy = - gtNewIconHandleNode(tree->AsIntCon()->gtIconVal, tree->gtFlags, tree->AsIntCon()->gtFieldSeq); - copy->AsIntCon()->gtCompileTimeHandle = tree->AsIntCon()->gtCompileTimeHandle; - copy->gtType = tree->gtType; - } - else -#endif - { - copy = gtNewIconNode(tree->AsIntCon()->gtIconVal, tree->gtType); -#ifdef DEBUG - copy->AsIntCon()->gtTargetHandle = tree->AsIntCon()->gtTargetHandle; -#endif - copy->AsIntCon()->gtCompileTimeHandle = tree->AsIntCon()->gtCompileTimeHandle; - copy->AsIntCon()->gtFieldSeq = tree->AsIntCon()->gtFieldSeq; - } - goto DONE; - - case GT_CNS_LNG: - copy = gtNewLconNode(tree->AsLngCon()->gtLconVal); - goto DONE; - - case GT_CNS_DBL: - { - copy = gtNewDconNode(tree->AsDblCon()->DconValue(), tree->TypeGet()); - goto DONE; - } - - case GT_CNS_STR: - copy = gtNewSconNode(tree->AsStrCon()->gtSconCPX, tree->AsStrCon()->gtScpHnd); - goto DONE; - -#if defined(FEATURE_SIMD) - case GT_CNS_VEC: - { - GenTreeVecCon* vecCon = gtNewVconNode(tree->TypeGet()); - vecCon->gtSimdVal = tree->AsVecCon()->gtSimdVal; - copy = vecCon; - goto DONE; - } -#endif // FEATURE_SIMD - -#if defined(FEATURE_MASKED_HW_INTRINSICS) - case GT_CNS_MSK: - { - GenTreeMskCon* mskCon = gtNewMskConNode(tree->TypeGet()); - mskCon->gtSimdMaskVal = tree->AsMskCon()->gtSimdMaskVal; - copy = mskCon; - goto DONE; - } -#endif // FEATURE_MASKED_HW_INTRINSICS - - case GT_LCL_VAR: - - // Remember that the local node has been cloned. The flag will be set on 'copy' as well. - tree->gtFlags |= GTF_VAR_MOREUSES; - copy = - gtNewLclvNode(tree->AsLclVar()->GetLclNum(), tree->gtType DEBUGARG(tree->AsLclVar()->gtLclILoffs)); - copy->AsLclVarCommon()->SetSsaNum(tree->AsLclVarCommon()->GetSsaNum()); - goto DONE; - - case GT_LCL_FLD: - // Remember that the local node has been cloned. The flag will be set on 'copy' as well. - tree->gtFlags |= GTF_VAR_MOREUSES; - copy = - new (this, GT_LCL_FLD) GenTreeLclFld(GT_LCL_FLD, tree->TypeGet(), tree->AsLclFld()->GetLclNum(), - tree->AsLclFld()->GetLclOffs(), tree->AsLclFld()->GetLayout()); - copy->AsLclFld()->SetSsaNum(tree->AsLclFld()->GetSsaNum()); - goto DONE; - - case GT_RET_EXPR: - // GT_RET_EXPR is unique node, that contains a link to a gtInlineCandidate node, - // that is part of another statement. We cannot clone both here and cannot - // create another GT_RET_EXPR that points to the same gtInlineCandidate. - NO_WAY("Cloning of GT_RET_EXPR node not supported"); - goto DONE; - - case GT_MEMORYBARRIER: - copy = new (this, GT_MEMORYBARRIER) GenTree(GT_MEMORYBARRIER, TYP_VOID); - goto DONE; - - case GT_FTN_ADDR: - copy = new (this, oper) GenTreeFptrVal(tree->gtType, tree->AsFptrVal()->gtFptrMethod); - copy->AsFptrVal()->gtFptrDelegateTarget = tree->AsFptrVal()->gtFptrDelegateTarget; - -#ifdef FEATURE_READYTORUN - copy->AsFptrVal()->gtEntryPoint = tree->AsFptrVal()->gtEntryPoint; -#endif - goto DONE; - - case GT_CATCH_ARG: - case GT_NO_OP: - case GT_NOP: - case GT_LABEL: - case GT_SWIFT_ERROR: - copy = new (this, oper) GenTree(oper, tree->gtType); - goto DONE; - -#if defined(FEATURE_EH_WINDOWS_X86) - case GT_END_LFIN: -#endif // FEATURE_EH_WINDOWS_X86 - case GT_JMP: - copy = new (this, oper) GenTreeVal(oper, tree->gtType, tree->AsVal()->gtVal1); - goto DONE; - - case GT_LCL_ADDR: - { - copy = new (this, oper) - GenTreeLclFld(oper, tree->TypeGet(), tree->AsLclFld()->GetLclNum(), tree->AsLclFld()->GetLclOffs()); - goto DONE; - } - - default: - NO_WAY("Cloning of node not supported"); - goto DONE; - } - } - - /* Is it a 'simple' unary/binary operator? */ - - if (kind & GTK_SMPOP) - { - /* If necessary, make sure we allocate a "fat" tree node */ - - switch (oper) - { - case GT_STORE_LCL_VAR: - // Remember that the local node has been cloned. The flag will be set on 'copy' as well. - tree->gtFlags |= GTF_VAR_MOREUSES; - copy = new (this, GT_STORE_LCL_VAR) - GenTreeLclVar(tree->TypeGet(), tree->AsLclVar()->GetLclNum(), tree->AsLclVar()->Data()); - break; - - case GT_STORE_LCL_FLD: - // Remember that the local node has been cloned. The flag will be set on 'copy' as well. - tree->gtFlags |= GTF_VAR_MOREUSES; - copy = new (this, GT_STORE_LCL_FLD) - GenTreeLclFld(tree->TypeGet(), tree->AsLclFld()->GetLclNum(), tree->AsLclFld()->GetLclOffs(), - tree->AsLclFld()->Data(), tree->AsLclFld()->GetLayout()); - break; - - /* These nodes sometimes get bashed to "fat" ones */ - - case GT_MUL: - case GT_DIV: - case GT_MOD: - - case GT_UDIV: - case GT_UMOD: - - // In the implementation of gtNewLargeOperNode you have - // to give an oper that will create a small node, - // otherwise it asserts. - // - if (GenTree::s_gtNodeSizes[oper] == TREE_NODE_SZ_SMALL) - { - copy = gtNewLargeOperNode(oper, tree->TypeGet(), tree->AsOp()->gtOp1, - tree->OperIsBinary() ? tree->AsOp()->gtOp2 : nullptr); - } - else // Always a large tree - { - if (tree->OperIsBinary()) - { - copy = gtNewOperNode(oper, tree->TypeGet(), tree->AsOp()->gtOp1, tree->AsOp()->gtOp2); - } - else - { - copy = gtNewOperNode(oper, tree->TypeGet(), tree->AsOp()->gtOp1); - } - } - break; - - case GT_CAST: - copy = new (this, LargeOpOpcode()) - GenTreeCast(tree->TypeGet(), tree->AsCast()->CastOp(), tree->IsUnsigned(), - tree->AsCast()->gtCastType DEBUGARG(/*largeNode*/ TRUE)); - break; - - case GT_INDEX_ADDR: - { - GenTreeIndexAddr* asIndAddr = tree->AsIndexAddr(); - - copy = new (this, GT_INDEX_ADDR) - GenTreeIndexAddr(asIndAddr->Arr(), asIndAddr->Index(), asIndAddr->gtElemType, - asIndAddr->gtStructElemClass, asIndAddr->gtElemSize, asIndAddr->gtLenOffset, - asIndAddr->gtElemOffset, asIndAddr->IsBoundsChecked()); - copy->AsIndexAddr()->gtIndRngFailBB = asIndAddr->gtIndRngFailBB; - } - break; - - case GT_ALLOCOBJ: - { - GenTreeAllocObj* asAllocObj = tree->AsAllocObj(); - copy = new (this, GT_ALLOCOBJ) - GenTreeAllocObj(tree->TypeGet(), asAllocObj->gtNewHelper, asAllocObj->gtHelperHasSideEffects, - asAllocObj->gtAllocObjClsHnd, asAllocObj->gtOp1); -#ifdef FEATURE_READYTORUN - copy->AsAllocObj()->gtEntryPoint = asAllocObj->gtEntryPoint; -#endif - } - break; - - case GT_RUNTIMELOOKUP: - { - GenTreeRuntimeLookup* asRuntimeLookup = tree->AsRuntimeLookup(); - - copy = new (this, GT_RUNTIMELOOKUP) - GenTreeRuntimeLookup(asRuntimeLookup->gtHnd, asRuntimeLookup->gtHndType, asRuntimeLookup->gtOp1); - } - break; - - case GT_STOREIND: - copy = new (this, GT_STOREIND) - GenTreeStoreInd(tree->TypeGet(), tree->AsIndir()->Addr(), tree->AsIndir()->Data()); - copy->AsStoreInd()->SetRMWStatus(tree->AsStoreInd()->GetRMWStatus()); - break; - - case GT_STORE_BLK: - copy = new (this, GT_STORE_BLK) GenTreeBlk(GT_STORE_BLK, tree->TypeGet(), tree->AsBlk()->Addr(), - tree->AsBlk()->Data(), tree->AsBlk()->GetLayout()); - break; - - case GT_ARR_ADDR: - copy = new (this, GT_ARR_ADDR) - GenTreeArrAddr(tree->AsArrAddr()->Addr(), tree->AsArrAddr()->GetElemType(), - tree->AsArrAddr()->GetElemClassHandle(), tree->AsArrAddr()->GetFirstElemOffset()); - break; - - case GT_ARR_LENGTH: - copy = - gtNewArrLen(tree->TypeGet(), tree->AsArrLen()->ArrRef(), tree->AsArrLen()->ArrLenOffset(), nullptr); - break; - - case GT_MDARR_LENGTH: - copy = - gtNewMDArrLen(tree->AsMDArr()->ArrRef(), tree->AsMDArr()->Dim(), tree->AsMDArr()->Rank(), nullptr); - break; - - case GT_MDARR_LOWER_BOUND: - copy = gtNewMDArrLowerBound(tree->AsMDArr()->ArrRef(), tree->AsMDArr()->Dim(), tree->AsMDArr()->Rank(), - nullptr); - break; - - case GT_QMARK: - copy = new (this, GT_QMARK) - GenTreeQmark(tree->TypeGet(), tree->AsOp()->gtGetOp1(), tree->AsOp()->gtGetOp2()->AsColon(), - tree->AsQmark()->ThenNodeLikelihood()); - break; - - case GT_BLK: - copy = new (this, GT_BLK) - GenTreeBlk(GT_BLK, tree->TypeGet(), tree->AsBlk()->Addr(), tree->AsBlk()->GetLayout()); - break; - - case GT_FIELD_ADDR: - copy = new (this, GT_FIELD_ADDR) - GenTreeFieldAddr(tree->TypeGet(), tree->AsFieldAddr()->GetFldObj(), tree->AsFieldAddr()->gtFldHnd, - tree->AsFieldAddr()->gtFldOffset); - copy->AsFieldAddr()->gtFldMayOverlap = tree->AsFieldAddr()->gtFldMayOverlap; -#ifdef FEATURE_READYTORUN - copy->AsFieldAddr()->gtFieldLookup = tree->AsFieldAddr()->gtFieldLookup; -#endif - copy->AsFieldAddr()->SetIsSpanLength(tree->AsFieldAddr()->IsSpanLength()); - break; - - case GT_BOX: - copy = new (this, GT_BOX) - GenTreeBox(tree->TypeGet(), tree->AsOp()->gtOp1, tree->AsBox()->gtDefStmtWhenInlinedBoxValue, - tree->AsBox()->gtCopyStmtWhenInlinedBoxValue); - tree->AsBox()->SetCloned(); - copy->AsBox()->SetCloned(); - break; - - case GT_INTRINSIC: - copy = new (this, GT_INTRINSIC) - GenTreeIntrinsic(tree->TypeGet(), tree->AsOp()->gtOp1, tree->AsOp()->gtOp2, - tree->AsIntrinsic()->gtIntrinsicName, - tree->AsIntrinsic()->gtMethodHandle R2RARG(tree->AsIntrinsic()->gtEntryPoint)); - break; - - case GT_BOUNDS_CHECK: - copy = new (this, GT_BOUNDS_CHECK) - GenTreeBoundsChk(tree->AsBoundsChk()->GetIndex(), tree->AsBoundsChk()->GetArrayLength(), - tree->AsBoundsChk()->gtThrowKind); - copy->AsBoundsChk()->gtIndRngFailBB = tree->AsBoundsChk()->gtIndRngFailBB; - copy->AsBoundsChk()->gtInxType = tree->AsBoundsChk()->gtInxType; - break; - - case GT_LEA: - { - GenTreeAddrMode* addrModeOp = tree->AsAddrMode(); - copy = new (this, GT_LEA) - GenTreeAddrMode(addrModeOp->TypeGet(), addrModeOp->Base(), addrModeOp->Index(), addrModeOp->gtScale, - static_cast(addrModeOp->Offset())); - } - break; - - case GT_COPY: - case GT_RELOAD: - { - copy = new (this, oper) GenTreeCopyOrReload(oper, tree->TypeGet(), tree->gtGetOp1()); - } - break; - - case GT_SWIFT_ERROR_RET: - copy = new (this, oper) GenTreeOp(oper, tree->TypeGet(), tree->gtGetOp1(), tree->gtGetOp2()); - break; - - default: - assert(!GenTree::IsExOp(tree->OperKind()) && tree->OperIsSimple()); - // We're in the SimpleOp case, so it's always unary or binary. - if (GenTree::OperIsUnary(tree->OperGet())) - { - copy = gtNewOperNode(oper, tree->TypeGet(), tree->AsOp()->gtOp1); - } - else - { - assert(GenTree::OperIsBinary(tree->OperGet())); - copy = gtNewOperNode(oper, tree->TypeGet(), tree->AsOp()->gtOp1, tree->AsOp()->gtOp2); - } - break; - } - - if (tree->AsOp()->gtOp1) - { - copy->AsOp()->gtOp1 = gtCloneExpr(tree->AsOp()->gtOp1); - } - - if (tree->gtGetOp2IfPresent()) - { - copy->AsOp()->gtOp2 = gtCloneExpr(tree->AsOp()->gtOp2); - } - - goto DONE; - } - - /* See what kind of a special operator we have here */ - - switch (oper) - { - case GT_CALL: - - // We can't safely clone calls that have GT_RET_EXPRs via gtCloneExpr. - // You must use gtCloneCandidateCall for these calls (and then do appropriate other fixup) - if (tree->AsCall()->IsInlineCandidate() || tree->AsCall()->IsGuardedDevirtualizationCandidate()) - { - NO_WAY("Cloning of calls with associated GT_RET_EXPR nodes is not supported"); - } - - copy = gtCloneExprCallHelper(tree->AsCall()); - break; - -#ifdef FEATURE_HW_INTRINSICS - case GT_HWINTRINSIC: - copy = new (this, GT_HWINTRINSIC) - GenTreeHWIntrinsic(tree->TypeGet(), IntrinsicNodeBuilder(getAllocator(CMK_ASTNode), tree->AsMultiOp()), - tree->AsHWIntrinsic()->GetHWIntrinsicId(), - tree->AsHWIntrinsic()->GetSimdBaseJitType(), tree->AsHWIntrinsic()->GetSimdSize()); - copy->AsHWIntrinsic()->SetAuxiliaryJitType(tree->AsHWIntrinsic()->GetAuxiliaryJitType()); - - if (tree->AsHWIntrinsic()->IsUserCall()) - { - copy->AsHWIntrinsic()->SetMethodHandle(this, tree->AsHWIntrinsic()->GetMethodHandle() - R2RARG(tree->AsHWIntrinsic()->GetEntryPoint())); - } - goto CLONE_MULTIOP_OPERANDS; -#endif -#if defined(FEATURE_SIMD) || defined(FEATURE_HW_INTRINSICS) - CLONE_MULTIOP_OPERANDS: - for (GenTree** use : copy->AsMultiOp()->UseEdges()) - { - *use = gtCloneExpr(*use); - } - break; -#endif - - case GT_ARR_ELEM: - { - GenTreeArrElem* arrElem = tree->AsArrElem(); - GenTree* inds[GT_ARR_MAX_RANK]; - for (unsigned dim = 0; dim < arrElem->gtArrRank; dim++) - { - inds[dim] = gtCloneExpr(arrElem->gtArrInds[dim]); - } - copy = new (this, GT_ARR_ELEM) GenTreeArrElem(arrElem->TypeGet(), gtCloneExpr(arrElem->gtArrObj), - arrElem->gtArrRank, arrElem->gtArrElemSize, &inds[0]); - } - break; - - case GT_PHI: - { - copy = new (this, GT_PHI) GenTreePhi(tree->TypeGet()); - GenTreePhi::Use** prevUse = ©->AsPhi()->gtUses; - for (GenTreePhi::Use& use : tree->AsPhi()->Uses()) - { - *prevUse = new (this, CMK_ASTNode) GenTreePhi::Use(gtCloneExpr(use.GetNode()), *prevUse); - prevUse = &((*prevUse)->NextRef()); - } - } - break; - - case GT_FIELD_LIST: - copy = new (this, GT_FIELD_LIST) GenTreeFieldList(); - for (GenTreeFieldList::Use& use : tree->AsFieldList()->Uses()) - { - copy->AsFieldList()->AddField(this, gtCloneExpr(use.GetNode()), use.GetOffset(), use.GetType()); - } - break; - - case GT_CMPXCHG: - copy = new (this, GT_CMPXCHG) - GenTreeCmpXchg(tree->TypeGet(), gtCloneExpr(tree->AsCmpXchg()->Addr()), - gtCloneExpr(tree->AsCmpXchg()->Data()), gtCloneExpr(tree->AsCmpXchg()->Comparand())); - break; - - case GT_SELECT: - copy = - new (this, oper) GenTreeConditional(oper, tree->TypeGet(), gtCloneExpr(tree->AsConditional()->gtCond), - gtCloneExpr(tree->AsConditional()->gtOp1), - gtCloneExpr(tree->AsConditional()->gtOp2)); - break; - default: -#ifdef DEBUG - gtDispTree(tree); -#endif - NO_WAY("unexpected operator"); - } - -DONE: - assert(copy->gtOper == oper); - - copy->gtVNPair = tree->gtVNPair; // A cloned tree gets the original's Value number pair - copy->gtFlags = tree->gtFlags; - -#if defined(DEBUG) - // Non-node debug flags should be propagated from 'tree' to 'copy' - copy->gtDebugFlags |= (tree->gtDebugFlags & ~GTF_DEBUG_NODE_MASK); -#endif - - /* Make sure to copy back fields that may have been initialized */ - - copy->CopyRawCosts(tree); - copy->CopyReg(tree); - return copy; -} - -//------------------------------------------------------------------------ -// InternalCopyFrom: -// Copy all information from the specified `CallArgs`, making these argument -// lists equivalent. Nodes are cloned using the specified function. -// -// Remarks: -// This function should not be used directly. Instead, use `gtCloneExpr` on -// the call node. -// -template -void CallArgs::InternalCopyFrom(Compiler* comp, CallArgs* other, CopyNodeFunc copyNode) -{ - assert((m_head == nullptr) && (m_lateHead == nullptr)); - - m_argsStackSize = other->m_argsStackSize; - m_hasThisPointer = other->m_hasThisPointer; - m_hasRetBuffer = other->m_hasRetBuffer; - m_isVarArgs = other->m_isVarArgs; - m_abiInformationDetermined = other->m_abiInformationDetermined; - m_hasRegArgs = other->m_hasRegArgs; - m_hasStackArgs = other->m_hasStackArgs; - m_argsComplete = other->m_argsComplete; - m_needsTemps = other->m_needsTemps; - - // Unix x86 flags related to stack alignment intentionally not copied as - // they depend on where the call will be inserted. - - CallArg** tail = &m_head; - for (CallArg& arg : other->Args()) - { - CallArg* carg = new (comp, CMK_CallArgs) CallArg(); - carg->m_earlyNode = arg.m_earlyNode != nullptr ? copyNode(arg.m_earlyNode) : nullptr; - carg->m_lateNode = arg.m_lateNode != nullptr ? copyNode(arg.m_lateNode) : nullptr; - carg->m_signatureClsHnd = arg.m_signatureClsHnd; - carg->m_signatureType = arg.m_signatureType; - carg->m_wellKnownArg = arg.m_wellKnownArg; - carg->m_needTmp = arg.m_needTmp; - carg->m_needPlace = arg.m_needPlace; - carg->m_processed = arg.m_processed; - carg->AbiInfo = arg.AbiInfo; - carg->NewAbiInfo = arg.NewAbiInfo; - *tail = carg; - tail = &carg->m_next; - } - - // Now copy late pointers. Note that these may not come in order. - tail = &m_lateHead; - for (CallArg& arg : other->LateArgs()) - { - CallArg* it = m_head; - CallArg* otherIt = other->m_head; - while (otherIt != &arg) - { - assert(it != nullptr && otherIt != nullptr); - it = it->m_next; - otherIt = otherIt->m_next; - } - - *tail = it; - tail = &it->m_lateNext; - } -} - -//------------------------------------------------------------------------ -// gtCloneExprCallHelper: clone a call tree -// -// Notes: -// Do not invoke this method directly, instead call either gtCloneExpr -// or gtCloneCandidateCall, as appropriate. -// -// Arguments: -// tree - the call to clone -// -// Returns: -// Cloned copy of call and all subtrees. - -GenTreeCall* Compiler::gtCloneExprCallHelper(GenTreeCall* tree) -{ - GenTreeCall* copy = new (this, GT_CALL) GenTreeCall(tree->TypeGet()); - - copy->gtCallMoreFlags = tree->gtCallMoreFlags; - INDEBUG(copy->gtCallDebugFlags = tree->gtCallDebugFlags); - - copy->gtArgs.InternalCopyFrom(this, &tree->gtArgs, [=](GenTree* node) { - return gtCloneExpr(node); - }); - - // The call sig comes from the EE and doesn't change throughout the compilation process, meaning - // we only really need one physical copy of it. Therefore a shallow pointer copy will suffice. - // (Note that this still holds even if the tree we are cloning was created by an inlinee compiler, - // because the inlinee still uses the inliner's memory allocator anyway.) - INDEBUG(copy->callSig = tree->callSig;) - - // The tail call info does not change after it is allocated, so for the same reasons as above - // a shallow copy suffices. - copy->tailCallInfo = tree->tailCallInfo; - - copy->gtRetClsHnd = tree->gtRetClsHnd; - copy->gtControlExpr = gtCloneExpr(tree->gtControlExpr); - copy->gtStubCallStubAddr = tree->gtStubCallStubAddr; - - /* Copy the union */ - if (tree->gtCallType == CT_INDIRECT) - { - copy->gtCallCookie = tree->gtCallCookie ? gtCloneExpr(tree->gtCallCookie) : nullptr; - copy->gtCallAddr = tree->gtCallAddr ? gtCloneExpr(tree->gtCallAddr) : nullptr; - } - else - { - copy->gtCallMethHnd = tree->gtCallMethHnd; - copy->gtInlineCandidateInfo = tree->gtInlineCandidateInfo; - copy->gtInlineInfoCount = tree->gtInlineInfoCount; - } - - copy->gtCallType = tree->gtCallType; - copy->gtReturnType = tree->gtReturnType; - -#if FEATURE_MULTIREG_RET - copy->gtReturnTypeDesc = tree->gtReturnTypeDesc; -#endif - -#ifdef FEATURE_READYTORUN - copy->setEntryPoint(tree->gtEntryPoint); -#endif - -#if defined(DEBUG) - copy->gtInlineObservation = tree->gtInlineObservation; - copy->gtRawILOffset = tree->gtRawILOffset; - copy->gtInlineContext = tree->gtInlineContext; -#endif - - copy->CopyOtherRegFlags(tree); - - // We keep track of the number of no return calls, so if we've cloned - // one of these, update the tracking. - // - if (tree->IsNoReturn()) - { - assert(copy->IsNoReturn()); - setMethodHasNoReturnCalls(); - } - - return copy; -} - -//------------------------------------------------------------------------ -// gtCloneCandidateCall: clone a call that is an inline or guarded -// devirtualization candidate (~ any call that can have a GT_RET_EXPR) -// -// Notes: -// If the call really is a candidate, the caller must take additional steps -// after cloning to re-establish candidate info and the relationship between -// the candidate and any associated GT_RET_EXPR. -// -// Arguments: -// call - the call to clone -// -// Returns: -// Cloned copy of call and all subtrees. - -GenTreeCall* Compiler::gtCloneCandidateCall(GenTreeCall* call) -{ - assert(call->IsInlineCandidate() || call->IsGuardedDevirtualizationCandidate()); - - GenTreeCall* result = gtCloneExprCallHelper(call); - - // There is some common post-processing in gtCloneExpr that we reproduce - // here, for the fields that make sense for candidate calls. - result->gtFlags |= call->gtFlags; - -#if defined(DEBUG) - result->gtDebugFlags |= (call->gtDebugFlags & ~GTF_DEBUG_NODE_MASK); -#endif - - result->CopyReg(call); - - return result; -} - -//------------------------------------------------------------------------ -// gtUpdateSideEffects: Update the side effects of a tree and its ancestors -// -// Arguments: -// stmt - The tree's statement -// tree - Tree to update the side effects for -// -// Note: If tree's order hasn't been established, the method updates side effect -// flags on all statement's nodes. - -void Compiler::gtUpdateSideEffects(Statement* stmt, GenTree* tree) -{ - if (fgNodeThreading == NodeThreading::AllTrees) - { - gtUpdateTreeAncestorsSideEffects(tree); - } - else - { - assert(fgNodeThreading != NodeThreading::LIR); - gtUpdateStmtSideEffects(stmt); - } -} - -//------------------------------------------------------------------------ -// gtUpdateTreeAncestorsSideEffects: Update the side effects of a tree and its ancestors -// when statement order has been established. -// -// Arguments: -// tree - Tree to update the side effects for -// -void Compiler::gtUpdateTreeAncestorsSideEffects(GenTree* tree) -{ - assert(fgNodeThreading == NodeThreading::AllTrees); - while (tree != nullptr) - { - gtUpdateNodeSideEffects(tree); - tree = tree->gtGetParent(nullptr); - } -} - -//------------------------------------------------------------------------ -// gtUpdateStmtSideEffects: Update the side effects for statement tree nodes. -// -// Arguments: -// stmt - The statement to update side effects on -// -void Compiler::gtUpdateStmtSideEffects(Statement* stmt) -{ - struct UpdateSideEffectsWalker : GenTreeVisitor - { - enum - { - DoPreOrder = true, - DoPostOrder = true, - }; - - UpdateSideEffectsWalker(Compiler* comp) - : GenTreeVisitor(comp) - { - } - - fgWalkResult PreOrderVisit(GenTree** use, GenTree* user) - { - GenTree* tree = *use; - tree->gtFlags &= ~(GTF_ASG | GTF_CALL | GTF_EXCEPT); - return WALK_CONTINUE; - } - - fgWalkResult PostOrderVisit(GenTree** use, GenTree* user) - { - GenTree* tree = *use; - - // Update the node's side effects first. - if (tree->OperMayThrow(m_compiler)) - { - tree->gtFlags |= GTF_EXCEPT; - } - - if (tree->OperRequiresAsgFlag()) - { - tree->gtFlags |= GTF_ASG; - } - - if (tree->OperRequiresCallFlag(m_compiler)) - { - tree->gtFlags |= GTF_CALL; - } - - // If this node is an indir or array meta-data load, and it doesn't have the GTF_EXCEPT bit set, we - // set the GTF_IND_NONFAULTING bit. This needs to be done after all children, and this node, have - // been processed. - if (tree->OperIsIndirOrArrMetaData() && ((tree->gtFlags & GTF_EXCEPT) == 0)) - { - tree->gtFlags |= GTF_IND_NONFAULTING; - } - - // Then update the parent's side effects based on this node. - if (user != nullptr) - { - user->gtFlags |= (tree->gtFlags & GTF_ALL_EFFECT); - } - return WALK_CONTINUE; - } - }; - - UpdateSideEffectsWalker walker(this); - walker.WalkTree(stmt->GetRootNodePointer(), nullptr); -} - -//------------------------------------------------------------------------ -// gtUpdateNodeOperSideEffects: Update the side effects based on the node operation. -// -// Arguments: -// tree - Tree to update the side effects on -// -// Notes: -// This method currently only updates GTF_EXCEPT, GTF_ASG, and GTF_CALL flags. -// The other side effect flags may remain unnecessarily (conservatively) set. -// The caller of this method is expected to update the flags based on the children's flags. -// -void Compiler::gtUpdateNodeOperSideEffects(GenTree* tree) -{ - if (tree->OperMayThrow(this)) - { - tree->gtFlags |= GTF_EXCEPT; - } - else - { - tree->gtFlags &= ~GTF_EXCEPT; - if (tree->OperIsIndirOrArrMetaData()) - { - tree->gtFlags |= GTF_IND_NONFAULTING; - } - } - - if (tree->OperRequiresAsgFlag()) - { - tree->gtFlags |= GTF_ASG; - } - else - { - tree->gtFlags &= ~GTF_ASG; - } - - if (tree->OperRequiresCallFlag(this)) - { - tree->gtFlags |= GTF_CALL; - } - else - { - tree->gtFlags &= ~GTF_CALL; - } -} - -//------------------------------------------------------------------------ -// gtUpdateNodeSideEffects: Update the side effects based on the node operation and -// children's side efects. -// -// Arguments: -// tree - Tree to update the side effects on -// -// Notes: -// This method currently only updates GTF_EXCEPT, GTF_ASG, and GTF_CALL flags. -// The other side effect flags may remain unnecessarily (conservatively) set. -// -void Compiler::gtUpdateNodeSideEffects(GenTree* tree) -{ - gtUpdateNodeOperSideEffects(tree); - tree->VisitOperands([tree](GenTree* operand) -> GenTree::VisitResult { - tree->gtFlags |= (operand->gtFlags & GTF_ALL_EFFECT); - return GenTree::VisitResult::Continue; - }); -} - -bool GenTree::gtSetFlags() const -{ - return (gtFlags & GTF_SET_FLAGS) != 0; -} - -GenTreeUseEdgeIterator::GenTreeUseEdgeIterator() - : m_advance(nullptr) - , m_node(nullptr) - , m_edge(nullptr) - , m_statePtr(nullptr) - , m_state(-1) -{ -} - -GenTreeUseEdgeIterator::GenTreeUseEdgeIterator(GenTree* node) - : m_advance(nullptr) - , m_node(node) - , m_edge(nullptr) - , m_statePtr(nullptr) - , m_state(0) -{ - assert(m_node != nullptr); - - // NOTE: the switch statement below must be updated when introducing new nodes. - - switch (m_node->OperGet()) - { - // Leaf nodes - case GT_LCL_VAR: - case GT_LCL_FLD: - case GT_LCL_ADDR: - case GT_CATCH_ARG: - case GT_LABEL: - case GT_FTN_ADDR: - case GT_RET_EXPR: - case GT_CNS_INT: - case GT_CNS_LNG: - case GT_CNS_DBL: - case GT_CNS_STR: -#if defined(FEATURE_SIMD) - case GT_CNS_VEC: -#endif // FEATURE_SIMD -#if defined(FEATURE_MASKED_HW_INTRINSICS) - case GT_CNS_MSK: -#endif // FEATURE_MASKED_HW_INTRINSICS - case GT_MEMORYBARRIER: - case GT_JMP: - case GT_JCC: - case GT_SETCC: - case GT_NO_OP: - case GT_START_NONGC: - case GT_START_PREEMPTGC: - case GT_PROF_HOOK: -#if defined(FEATURE_EH_WINDOWS_X86) - case GT_END_LFIN: -#endif // FEATURE_EH_WINDOWS_X86 - case GT_PHI_ARG: - case GT_JMPTABLE: - case GT_PHYSREG: - case GT_EMITNOP: - case GT_PINVOKE_PROLOG: - case GT_PINVOKE_EPILOG: - case GT_IL_OFFSET: - case GT_NOP: - case GT_SWIFT_ERROR: - m_state = -1; - return; - - // Standard unary operators - case GT_STORE_LCL_VAR: - case GT_STORE_LCL_FLD: - case GT_NOT: - case GT_NEG: - case GT_COPY: - case GT_RELOAD: - case GT_ARR_LENGTH: - case GT_MDARR_LENGTH: - case GT_MDARR_LOWER_BOUND: - case GT_CAST: - case GT_BITCAST: - case GT_CKFINITE: - case GT_LCLHEAP: - case GT_IND: - case GT_BLK: - case GT_BOX: - case GT_ALLOCOBJ: - case GT_RUNTIMELOOKUP: - case GT_ARR_ADDR: - case GT_INIT_VAL: - case GT_JTRUE: - case GT_SWITCH: - case GT_NULLCHECK: - case GT_PUTARG_REG: - case GT_PUTARG_STK: - case GT_BSWAP: - case GT_BSWAP16: - case GT_KEEPALIVE: - case GT_INC_SATURATE: -#if FEATURE_ARG_SPLIT - case GT_PUTARG_SPLIT: -#endif // FEATURE_ARG_SPLIT - case GT_RETURNTRAP: - m_edge = &m_node->AsUnOp()->gtOp1; - assert(*m_edge != nullptr); - m_advance = &GenTreeUseEdgeIterator::Terminate; - return; - - // Unary operators with an optional operand - case GT_FIELD_ADDR: - case GT_RETURN: - case GT_RETFILT: - if (m_node->AsUnOp()->gtOp1 == nullptr) - { - assert(m_node->NullOp1Legal()); - m_state = -1; - } - else - { - m_edge = &m_node->AsUnOp()->gtOp1; - m_advance = &GenTreeUseEdgeIterator::Terminate; - } - return; - -// Variadic nodes -#ifdef FEATURE_HW_INTRINSICS - case GT_HWINTRINSIC: - SetEntryStateForMultiOp(); - return; -#endif // FEATURE_HW_INTRINSICS - - // LEA, which may have no first operand - case GT_LEA: - if (m_node->AsAddrMode()->gtOp1 == nullptr) - { - m_edge = &m_node->AsAddrMode()->gtOp2; - m_advance = &GenTreeUseEdgeIterator::Terminate; - } - else - { - SetEntryStateForBinOp(); - } - return; - - // Special nodes - case GT_FIELD_LIST: - m_statePtr = m_node->AsFieldList()->Uses().GetHead(); - m_advance = &GenTreeUseEdgeIterator::AdvanceFieldList; - AdvanceFieldList(); - return; - - case GT_PHI: - m_statePtr = m_node->AsPhi()->gtUses; - m_advance = &GenTreeUseEdgeIterator::AdvancePhi; - AdvancePhi(); - return; - - case GT_CMPXCHG: - m_edge = &m_node->AsCmpXchg()->Addr(); - assert(*m_edge != nullptr); - m_advance = &GenTreeUseEdgeIterator::AdvanceCmpXchg; - return; - - case GT_ARR_ELEM: - m_edge = &m_node->AsArrElem()->gtArrObj; - assert(*m_edge != nullptr); - m_advance = &GenTreeUseEdgeIterator::AdvanceArrElem; - return; - - case GT_CALL: - m_statePtr = m_node->AsCall()->gtArgs.Args().begin().GetArg(); - m_advance = &GenTreeUseEdgeIterator::AdvanceCall; - AdvanceCall(); - return; -#ifdef TARGET_ARM64 - case GT_SELECT_NEG: - case GT_SELECT_INV: - case GT_SELECT_INC: -#endif - case GT_SELECT: - m_edge = &m_node->AsConditional()->gtCond; - assert(*m_edge != nullptr); - m_advance = &GenTreeUseEdgeIterator::AdvanceConditional; - return; - - // Binary nodes - default: - assert(m_node->OperIsBinary()); - SetEntryStateForBinOp(); - return; - } -} - -//------------------------------------------------------------------------ -// GenTreeUseEdgeIterator::AdvanceCmpXchg: produces the next operand of a CmpXchg node and advances the state. -// -void GenTreeUseEdgeIterator::AdvanceCmpXchg() -{ - switch (m_state) - { - case 0: - m_edge = &m_node->AsCmpXchg()->Data(); - m_state = 1; - break; - case 1: - m_edge = &m_node->AsCmpXchg()->Comparand(); - m_advance = &GenTreeUseEdgeIterator::Terminate; - break; - default: - unreached(); - } - - assert(*m_edge != nullptr); -} - -//------------------------------------------------------------------------ -// GenTreeUseEdgeIterator::AdvanceArrElem: produces the next operand of a ArrElem node and advances the state. -// -// Because these nodes are variadic, this function uses `m_state` to index into the list of array indices. -// -void GenTreeUseEdgeIterator::AdvanceArrElem() -{ - if (m_state < m_node->AsArrElem()->gtArrRank) - { - m_edge = &m_node->AsArrElem()->gtArrInds[m_state]; - assert(*m_edge != nullptr); - m_state++; - } - else - { - m_state = -1; - } -} - -//------------------------------------------------------------------------ -// GenTreeUseEdgeIterator::AdvanceFieldList: produces the next operand of a FieldList node and advances the state. -// -void GenTreeUseEdgeIterator::AdvanceFieldList() -{ - assert(m_state == 0); - - if (m_statePtr == nullptr) - { - m_state = -1; - } - else - { - GenTreeFieldList::Use* currentUse = static_cast(m_statePtr); - m_edge = ¤tUse->NodeRef(); - m_statePtr = currentUse->GetNext(); - } -} - -//------------------------------------------------------------------------ -// GenTreeUseEdgeIterator::AdvancePhi: produces the next operand of a Phi node and advances the state. -// -void GenTreeUseEdgeIterator::AdvancePhi() -{ - assert(m_state == 0); - - if (m_statePtr == nullptr) - { - m_state = -1; - } - else - { - GenTreePhi::Use* currentUse = static_cast(m_statePtr); - m_edge = ¤tUse->NodeRef(); - m_statePtr = currentUse->GetNext(); - } -} - -//------------------------------------------------------------------------ -// GenTreeUseEdgeIterator::AdvanceConditional: produces the next operand of a conditional node and advances the state. -// -void GenTreeUseEdgeIterator::AdvanceConditional() -{ - GenTreeConditional* const conditional = m_node->AsConditional(); - switch (m_state) - { - case 0: - m_edge = &conditional->gtOp1; - if (conditional->gtOp2 == nullptr) - { - m_advance = &GenTreeUseEdgeIterator::Terminate; - } - else - { - m_state = 1; - } - break; - case 1: - m_edge = &conditional->gtOp2; - m_advance = &GenTreeUseEdgeIterator::Terminate; - break; - default: - unreached(); - } - - assert(*m_edge != nullptr); -} - -//------------------------------------------------------------------------ -// GenTreeUseEdgeIterator::AdvanceBinOp: produces the next operand of a binary node and advances the state. -// -// This function must be instantiated s.t. `ReverseOperands` is `true` iff the node is marked with the -// `GTF_REVERSE_OPS` flag. -// -template -void GenTreeUseEdgeIterator::AdvanceBinOp() -{ - assert(ReverseOperands == ((m_node->gtFlags & GTF_REVERSE_OPS) != 0)); - - m_edge = !ReverseOperands ? &m_node->AsOp()->gtOp2 : &m_node->AsOp()->gtOp1; - assert(*m_edge != nullptr); - m_advance = &GenTreeUseEdgeIterator::Terminate; -} - -//------------------------------------------------------------------------ -// GenTreeUseEdgeIterator::SetEntryStateForBinOp: produces the first operand of a binary node and chooses -// the appropriate advance function. -// -void GenTreeUseEdgeIterator::SetEntryStateForBinOp() -{ - assert(m_node != nullptr); - assert(m_node->OperIsBinary()); - - GenTreeOp* const node = m_node->AsOp(); - - if (node->gtOp2 == nullptr) - { - assert(node->gtOp1 != nullptr); - assert(node->NullOp2Legal()); - m_edge = &node->gtOp1; - m_advance = &GenTreeUseEdgeIterator::Terminate; - } - else if ((node->gtFlags & GTF_REVERSE_OPS) != 0) - { - m_edge = &m_node->AsOp()->gtOp2; - m_advance = &GenTreeUseEdgeIterator::AdvanceBinOp; - } - else - { - m_edge = &m_node->AsOp()->gtOp1; - m_advance = &GenTreeUseEdgeIterator::AdvanceBinOp; - } -} - -#if defined(FEATURE_SIMD) || defined(FEATURE_HW_INTRINSICS) -//------------------------------------------------------------------------ -// GenTreeUseEdgeIterator::AdvanceMultiOp: produces the next operand of a multi-op node and advances the state. -// -// Takes advantage of the fact that GenTreeMultiOp stores the operands in a contiguous array, simply -// incrementing the "m_edge" pointer, unless the end, stored in "m_statePtr", has been reached. -// -void GenTreeUseEdgeIterator::AdvanceMultiOp() -{ - assert(m_node != nullptr); - assert(m_node->OperIs(GT_HWINTRINSIC)); - - m_edge++; - if (m_edge == m_statePtr) - { - Terminate(); - } -} - -//------------------------------------------------------------------------ -// GenTreeUseEdgeIterator::AdvanceReversedMultiOp: produces the next operand of a multi-op node -// marked with GTF_REVERSE_OPS and advances the state. -// -// Takes advantage of the fact that GenTreeMultiOp stores the operands in a contiguous array, simply -// decrementing the "m_edge" pointer, unless the beginning, stored in "m_statePtr", has been reached. -// -void GenTreeUseEdgeIterator::AdvanceReversedMultiOp() -{ - assert(m_node != nullptr); - assert(m_node->OperIs(GT_HWINTRINSIC)); - assert((m_node->AsMultiOp()->GetOperandCount() == 2) && m_node->IsReverseOp()); - - m_edge--; - if (m_edge == m_statePtr) - { - Terminate(); - } -} - -//------------------------------------------------------------------------ -// GenTreeUseEdgeIterator::SetEntryStateForMultiOp: produces the first operand of a multi-op node and sets the -// required advance function. -// -void GenTreeUseEdgeIterator::SetEntryStateForMultiOp() -{ - size_t operandCount = m_node->AsMultiOp()->GetOperandCount(); - - if (operandCount == 0) - { - Terminate(); - } - else - { - if (m_node->IsReverseOp()) - { - assert(operandCount == 2); - - m_edge = m_node->AsMultiOp()->GetOperandArray() + 1; - m_statePtr = m_node->AsMultiOp()->GetOperandArray() - 1; - m_advance = &GenTreeUseEdgeIterator::AdvanceReversedMultiOp; - } - else - { - m_edge = m_node->AsMultiOp()->GetOperandArray(); - m_statePtr = m_node->AsMultiOp()->GetOperandArray(operandCount); - m_advance = &GenTreeUseEdgeIterator::AdvanceMultiOp; - } - } -} -#endif - -//------------------------------------------------------------------------ -// GenTreeUseEdgeIterator::AdvanceCall: produces the next operand of a call node and advances the state. -// -// This function is a bit tricky: in order to avoid doing unnecessary work, it is instantiated with the -// state number the iterator will be in when it is called. For example, `AdvanceCall` -// is the instantiation used when the iterator is at the `CALL_INSTANCE` state (i.e. the entry state). -// This sort of templating allows each state to avoid processing earlier states without unnecessary -// duplication of code. -// -// Note that this method expands the argument list (`gtArgs.Args()` and `gtArgs.LateArgs()`) into their -// component operands. -// -template -void GenTreeUseEdgeIterator::AdvanceCall() -{ - GenTreeCall* const call = m_node->AsCall(); - - switch (state) - { - case CALL_ARGS: - while (m_statePtr != nullptr) - { - CallArg* arg = static_cast(m_statePtr); - m_edge = &arg->EarlyNodeRef(); - m_statePtr = arg->GetNext(); - - if (*m_edge != nullptr) - { - return; - } - } - m_statePtr = call->gtArgs.LateArgs().begin().GetArg(); - m_advance = &GenTreeUseEdgeIterator::AdvanceCall; - FALLTHROUGH; - - case CALL_LATE_ARGS: - if (m_statePtr != nullptr) - { - CallArg* arg = static_cast(m_statePtr); - m_edge = &arg->LateNodeRef(); - assert(*m_edge != nullptr); - m_statePtr = arg->GetLateNext(); - return; - } - m_advance = &GenTreeUseEdgeIterator::AdvanceCall; - FALLTHROUGH; - - case CALL_CONTROL_EXPR: - if (call->gtControlExpr != nullptr) - { - if (call->gtCallType == CT_INDIRECT) - { - m_advance = &GenTreeUseEdgeIterator::AdvanceCall; - } - else - { - m_advance = &GenTreeUseEdgeIterator::Terminate; - } - m_edge = &call->gtControlExpr; - return; - } - else if (call->gtCallType != CT_INDIRECT) - { - m_state = -1; - return; - } - FALLTHROUGH; - - case CALL_COOKIE: - assert(call->gtCallType == CT_INDIRECT); - - m_advance = &GenTreeUseEdgeIterator::AdvanceCall; - if (call->gtCallCookie != nullptr) - { - m_edge = &call->gtCallCookie; - return; - } - FALLTHROUGH; - - case CALL_ADDRESS: - assert(call->gtCallType == CT_INDIRECT); - - m_advance = &GenTreeUseEdgeIterator::Terminate; - if (call->gtCallAddr != nullptr) - { - m_edge = &call->gtCallAddr; - } - return; - - default: - unreached(); - } -} - -//------------------------------------------------------------------------ -// GenTreeUseEdgeIterator::Terminate: advances the iterator to the terminal state. -// -void GenTreeUseEdgeIterator::Terminate() -{ - m_state = -1; -} - -//------------------------------------------------------------------------ -// GenTreeUseEdgeIterator::operator++: advances the iterator to the next operand. -// -GenTreeUseEdgeIterator& GenTreeUseEdgeIterator::operator++() -{ - // If we've reached the terminal state, do nothing. - if (m_state != -1) - { - (this->*m_advance)(); - } - - return *this; -} - -GenTreeUseEdgeIterator GenTree::UseEdgesBegin() -{ - return GenTreeUseEdgeIterator(this); -} - -GenTreeUseEdgeIterator GenTree::UseEdgesEnd() -{ - return GenTreeUseEdgeIterator(); -} - -IteratorPair GenTree::UseEdges() -{ - return MakeIteratorPair(UseEdgesBegin(), UseEdgesEnd()); -} - -GenTreeOperandIterator GenTree::OperandsBegin() -{ - return GenTreeOperandIterator(this); -} - -GenTreeOperandIterator GenTree::OperandsEnd() -{ - return GenTreeOperandIterator(); -} - -IteratorPair GenTree::Operands() -{ - return MakeIteratorPair(OperandsBegin(), OperandsEnd()); -} - -bool GenTree::Precedes(GenTree* other) -{ - assert(other != nullptr); - - for (GenTree* node = gtNext; node != nullptr; node = node->gtNext) - { - if (node == other) - { - return true; - } - } - - return false; -} - -//------------------------------------------------------------------------------ -// SetIndirExceptionFlags : Set GTF_EXCEPT and GTF_IND_NONFAULTING flags as appropriate -// on an indirection or an array length node. -// -// Arguments: -// comp - compiler instance -// -void GenTree::SetIndirExceptionFlags(Compiler* comp) -{ - assert(OperIsIndirOrArrMetaData() && (OperIsSimple() || OperIs(GT_CMPXCHG))); - - if (IndirMayFault(comp)) - { - gtFlags |= GTF_EXCEPT; - return; - } - - GenTree* addr = GetIndirOrArrMetaDataAddr(); - - gtFlags |= GTF_IND_NONFAULTING; - gtFlags &= ~GTF_EXCEPT; - gtFlags |= addr->gtFlags & GTF_EXCEPT; - if (OperIsBinary()) - { - gtFlags |= gtGetOp2()->gtFlags & GTF_EXCEPT; - } - else if (OperIs(GT_CMPXCHG)) - { - gtFlags |= AsCmpXchg()->Data()->gtFlags & GTF_EXCEPT; - gtFlags |= AsCmpXchg()->Comparand()->gtFlags & GTF_EXCEPT; - } -} - -//------------------------------------------------------------------------------ -// HandleKindDataIsInvariant: Returns true if the data referred to by a handle -// address is guaranteed to be invariant. -// -// Arguments: -// flags - GenTree flags for handle. -// -/* static */ -bool GenTree::HandleKindDataIsInvariant(GenTreeFlags flags) -{ - GenTreeFlags handleKind = flags & GTF_ICON_HDL_MASK; - assert(handleKind != GTF_EMPTY); - - switch (handleKind) - { - case GTF_ICON_SCOPE_HDL: - case GTF_ICON_CLASS_HDL: - case GTF_ICON_METHOD_HDL: - case GTF_ICON_FIELD_HDL: - case GTF_ICON_STR_HDL: - case GTF_ICON_CONST_PTR: - case GTF_ICON_VARG_HDL: - case GTF_ICON_PINVKI_HDL: - case GTF_ICON_TOKEN_HDL: - case GTF_ICON_TLS_HDL: - case GTF_ICON_CIDMID_HDL: - case GTF_ICON_FIELD_SEQ: - case GTF_ICON_STATIC_ADDR_PTR: - case GTF_ICON_SECREL_OFFSET: - case GTF_ICON_TLSGD_OFFSET: - return true; - case GTF_ICON_FTN_ADDR: - case GTF_ICON_GLOBAL_PTR: - case GTF_ICON_STATIC_HDL: - case GTF_ICON_BBC_PTR: - case GTF_ICON_STATIC_BOX_PTR: - default: - return false; - } -} - -#ifdef DEBUG - -/* static */ int GenTree::gtDispFlags(GenTreeFlags flags, GenTreeDebugFlags debugFlags) -{ - int charsDisplayed = 10; // the "baseline" number of flag characters displayed - - printf("%c", (flags & GTF_ASG) ? 'A' : (IsContained(flags) ? 'c' : '-')); - printf("%c", (flags & GTF_CALL) ? 'C' : '-'); - printf("%c", (flags & GTF_EXCEPT) ? 'X' : '-'); - printf("%c", (flags & GTF_GLOB_REF) ? 'G' : '-'); - printf("%c", (debugFlags & GTF_DEBUG_NODE_MORPHED) ? '+' : // First print '+' if GTF_DEBUG_NODE_MORPHED is set - (flags & GTF_ORDER_SIDEEFF) ? 'O' - : '-'); // otherwise print 'O' or '-' - printf("%c", (flags & GTF_COLON_COND) ? '?' : '-'); - printf("%c", (flags & GTF_DONT_CSE) ? 'N' : // N is for No cse - (flags & GTF_MAKE_CSE) ? 'H' - : '-'); // H is for Hoist this expr - printf("%c", (flags & GTF_REVERSE_OPS) ? 'R' : '-'); - printf("%c", (flags & GTF_UNSIGNED) ? 'U' : '-'); - - // Both GTF_SPILL and GTF_SPILLED: '#' - // Only GTF_SPILLED: 'z' - // Only GTF_SPILL: 'Z' - printf("%c", ((flags & (GTF_SPILL | GTF_SPILLED)) == (GTF_SPILL | GTF_SPILLED)) - ? '#' - : ((flags & GTF_SPILLED) ? 'z' : ((flags & GTF_SPILL) ? 'Z' : '-'))); - - return charsDisplayed; -} - -/* static */ -const char* GenTree::gtGetHandleKindString(GenTreeFlags flags) -{ - GenTreeFlags handleKind = flags & GTF_ICON_HDL_MASK; - switch (handleKind) - { - case 0: - return ""; - case GTF_ICON_SCOPE_HDL: - return "GTF_ICON_SCOPE_HDL"; - case GTF_ICON_CLASS_HDL: - return "GTF_ICON_CLASS_HDL"; - case GTF_ICON_METHOD_HDL: - return "GTF_ICON_METHOD_HDL"; - case GTF_ICON_FIELD_HDL: - return "GTF_ICON_FIELD_HDL"; - case GTF_ICON_STATIC_HDL: - return "GTF_ICON_STATIC_HDL"; - case GTF_ICON_STR_HDL: - return "GTF_ICON_STR_HDL"; - case GTF_ICON_OBJ_HDL: - return "GTF_ICON_OBJ_HDL"; - case GTF_ICON_CONST_PTR: - return "GTF_ICON_CONST_PTR"; - case GTF_ICON_GLOBAL_PTR: - return "GTF_ICON_GLOBAL_PTR"; - case GTF_ICON_VARG_HDL: - return "GTF_ICON_VARG_HDL"; - case GTF_ICON_PINVKI_HDL: - return "GTF_ICON_PINVKI_HDL"; - case GTF_ICON_TOKEN_HDL: - return "GTF_ICON_TOKEN_HDL"; - case GTF_ICON_TLS_HDL: - return "GTF_ICON_TLS_HDL"; - case GTF_ICON_FTN_ADDR: - return "GTF_ICON_FTN_ADDR"; - case GTF_ICON_CIDMID_HDL: - return "GTF_ICON_CIDMID_HDL"; - case GTF_ICON_BBC_PTR: - return "GTF_ICON_BBC_PTR"; - case GTF_ICON_STATIC_BOX_PTR: - return "GTF_ICON_STATIC_BOX_PTR"; - case GTF_ICON_FIELD_SEQ: - return "GTF_ICON_FIELD_SEQ"; - case GTF_ICON_STATIC_ADDR_PTR: - return "GTF_ICON_STATIC_ADDR_PTR"; - case GTF_ICON_SECREL_OFFSET: - return "GTF_ICON_SECREL_OFFSET"; - case GTF_ICON_TLSGD_OFFSET: - return "GTF_ICON_TLSGD_OFFSET"; - default: - return "ILLEGAL!"; - } -} - -#ifdef TARGET_X86 -inline const char* GetCallConvName(CorInfoCallConvExtension callConv) -{ - switch (callConv) - { - case CorInfoCallConvExtension::Managed: - return "Managed"; - case CorInfoCallConvExtension::C: - return "C"; - case CorInfoCallConvExtension::Stdcall: - return "Stdcall"; - case CorInfoCallConvExtension::Thiscall: - return "Thiscall"; - case CorInfoCallConvExtension::Fastcall: - return "Fastcall"; - case CorInfoCallConvExtension::CMemberFunction: - return "CMemberFunction"; - case CorInfoCallConvExtension::StdcallMemberFunction: - return "StdcallMemberFunction"; - case CorInfoCallConvExtension::FastcallMemberFunction: - return "FastcallMemberFunction"; - default: - return "UnknownCallConv"; - } -} -#endif // TARGET_X86 - -/*****************************************************************************/ - -void Compiler::gtDispNodeName(GenTree* tree) -{ - /* print the node name */ - - const char* name; - - assert(tree); - if (tree->gtOper < GT_COUNT) - { - name = GenTree::OpName(tree->OperGet()); - } - else - { - name = ""; - } - char buf[32]; - char* bufp = &buf[0]; - - if (tree->IsIconHandle()) - { - sprintf_s(bufp, sizeof(buf), " %s(h)%c", name, 0); - } - else if (tree->gtOper == GT_PUTARG_STK) - { - sprintf_s(bufp, sizeof(buf), " %s [+0x%02x]%c", name, tree->AsPutArgStk()->getArgOffset(), 0); - } - else if (tree->gtOper == GT_CALL) - { - const char* callType = "CALL"; - const char* gtfType = ""; - const char* ctType = ""; - char gtfTypeBuf[100]; - - if (tree->AsCall()->gtCallType == CT_USER_FUNC) - { - if (tree->AsCall()->IsVirtual()) - { - callType = "CALLV"; - } - } - else if (tree->AsCall()->IsHelperCall()) - { - ctType = " help"; - } - else if (tree->AsCall()->gtCallType == CT_INDIRECT) - { - ctType = " ind"; - } - else - { - assert(!"Unknown gtCallType"); - } - - if (tree->gtFlags & GTF_CALL_NULLCHECK) - { - gtfType = " nullcheck"; - } - if (tree->AsCall()->IsVirtualVtable()) - { - gtfType = " vt-ind"; - } - else if (tree->AsCall()->IsVirtualStub()) - { - gtfType = " stub"; - } -#ifdef FEATURE_READYTORUN - else if (tree->AsCall()->IsR2RRelativeIndir()) - { - gtfType = " r2r_ind"; - } - else if (tree->gtFlags & GTF_TLS_GET_ADDR) - { - gtfType = " _tls_get_addr"; - } -#endif // FEATURE_READYTORUN - else if (tree->gtFlags & GTF_CALL_UNMANAGED) - { - char* gtfTypeBufWalk = gtfTypeBuf; - gtfTypeBufWalk += SimpleSprintf_s(gtfTypeBufWalk, gtfTypeBuf, sizeof(gtfTypeBuf), " unman"); - if (tree->gtFlags & GTF_CALL_POP_ARGS) - { - gtfTypeBufWalk += SimpleSprintf_s(gtfTypeBufWalk, gtfTypeBuf, sizeof(gtfTypeBuf), " popargs"); - } -#ifdef TARGET_X86 - gtfTypeBufWalk += SimpleSprintf_s(gtfTypeBufWalk, gtfTypeBuf, sizeof(gtfTypeBuf), " %s", - GetCallConvName(tree->AsCall()->GetUnmanagedCallConv())); -#endif // TARGET_X86 - gtfType = gtfTypeBuf; - } - - sprintf_s(bufp, sizeof(buf), " %s%s%s%c", callType, ctType, gtfType, 0); - } - else if (tree->gtOper == GT_ARR_ELEM) - { - bufp += SimpleSprintf_s(bufp, buf, sizeof(buf), " %s[", name); - for (unsigned rank = tree->AsArrElem()->gtArrRank - 1; rank; rank--) - { - bufp += SimpleSprintf_s(bufp, buf, sizeof(buf), ","); - } - SimpleSprintf_s(bufp, buf, sizeof(buf), "]"); - } - else if (tree->gtOper == GT_LEA) - { - GenTreeAddrMode* lea = tree->AsAddrMode(); - bufp += SimpleSprintf_s(bufp, buf, sizeof(buf), " %s(", name); - if (lea->HasBase()) - { - bufp += SimpleSprintf_s(bufp, buf, sizeof(buf), "b+"); - } - if (lea->HasIndex()) - { - bufp += SimpleSprintf_s(bufp, buf, sizeof(buf), "(i*%d)+", lea->gtScale); - } - bufp += SimpleSprintf_s(bufp, buf, sizeof(buf), "%d)", lea->Offset()); - } - else if (tree->gtOper == GT_BOUNDS_CHECK) - { - switch (tree->AsBoundsChk()->gtThrowKind) - { - case SCK_RNGCHK_FAIL: - { - bufp += SimpleSprintf_s(bufp, buf, sizeof(buf), " %s_Rng", name); - if (tree->AsBoundsChk()->gtIndRngFailBB != nullptr) - { - bufp += SimpleSprintf_s(bufp, buf, sizeof(buf), " -> " FMT_BB, - tree->AsBoundsChk()->gtIndRngFailBB->bbNum); - } - break; - } - case SCK_ARG_EXCPN: - sprintf_s(bufp, sizeof(buf), " %s_Arg", name); - break; - case SCK_ARG_RNG_EXCPN: - sprintf_s(bufp, sizeof(buf), " %s_ArgRng", name); - break; - default: - unreached(); - } - } - else if (tree->gtOverflowEx()) - { - sprintf_s(bufp, sizeof(buf), " %s_ovfl%c", name, 0); - } - else - { - sprintf_s(bufp, sizeof(buf), " %s%c", name, 0); - } - - if (strlen(buf) < 10) - { - printf(" %-10s", buf); - } - else - { - printf(" %s", buf); - } -} - -//------------------------------------------------------------------------ -// gtDispVN: Utility function that prints a tree's ValueNumber: gtVNPair -// -void Compiler::gtDispVN(GenTree* tree) -{ - if (tree->gtVNPair.GetLiberal() != ValueNumStore::NoVN) - { - assert(tree->gtVNPair.GetConservative() != ValueNumStore::NoVN); - printf(" "); - vnpPrint(tree->gtVNPair, 0); - } -} - -//------------------------------------------------------------------------ -// gtDispCommonEndLine -// Utility function that prints the following node information -// 1: The associated zero field sequence (if any) -// 2. The register assigned to this node (if any) -// 2. The value number assigned (if any) -// 3. A newline character -// -void Compiler::gtDispCommonEndLine(GenTree* tree) -{ - gtDispRegVal(tree); - gtDispVN(tree); - printf("\n"); -} - -//------------------------------------------------------------------------ -// gtDispNode: Print a tree to jitstdout. -// -// Arguments: -// tree - the tree to be printed -// indentStack - the specification for the current level of indentation & arcs -// msg - a contextual method (i.e. from the parent) to print -// -// Return Value: -// None. -// -// Notes: -// 'indentStack' may be null, in which case no indentation or arcs are printed -// 'msg' may be null - -void Compiler::gtDispNode(GenTree* tree, IndentStack* indentStack, _In_ _In_opt_z_ const char* msg, bool isLIR) -{ - bool printFlags = true; // always true.. - - int msgLength = 35; - - GenTree* prev; - - if (tree->gtSeqNum) - { - printf("N%03u ", tree->gtSeqNum); - if (tree->gtCostsInitialized) - { - printf("(%3u,%3u) ", tree->GetCostEx(), tree->GetCostSz()); - } - else - { - printf("(???" - ",???" - ") "); // This probably indicates a bug: the node has a sequence number, but not costs. - } - } - else - { - prev = tree; - - bool hasSeqNum = true; - unsigned dotNum = 0; - do - { - dotNum++; - prev = prev->gtPrev; - - if ((prev == nullptr) || (prev == tree)) - { - hasSeqNum = false; - break; - } - - assert(prev); - } while (prev->gtSeqNum == 0); - - // If we have an indent stack, don't add additional characters, - // as it will mess up the alignment. - bool displayDotNum = hasSeqNum && (indentStack == nullptr); - if (displayDotNum) - { - printf("N%03u.%02u ", prev->gtSeqNum, dotNum); - } - else - { - printf(" "); - } - - if (tree->gtCostsInitialized) - { - printf("(%3u,%3u) ", tree->GetCostEx(), tree->GetCostSz()); - } - else - { - if (displayDotNum) - { - // Do better alignment in this case - printf(" "); - } - else - { - printf(" "); - } - } - } - - if (optValnumCSE_phase) - { - if (IS_CSE_INDEX(tree->gtCSEnum)) - { - printf(FMT_CSE " (%s)", GET_CSE_INDEX(tree->gtCSEnum), (IS_CSE_USE(tree->gtCSEnum) ? "use" : "def")); - } - else - { - printf(" "); - } - } - - /* Print the node ID */ - printTreeID(JitConfig.JitDumpTreeIDs() ? tree : nullptr); - printf(" "); - - if (tree->gtOper >= GT_COUNT) - { - printf(" **** ILLEGAL NODE ****"); - return; - } - - if (printFlags) - { - /* First print the flags associated with the node */ - switch (tree->gtOper) - { - case GT_BLK: - case GT_IND: - case GT_STOREIND: - case GT_STORE_BLK: - // We prefer printing V or U - if ((tree->gtFlags & (GTF_IND_VOLATILE | GTF_IND_UNALIGNED)) == 0) - { - if (tree->gtFlags & GTF_IND_TGT_NOT_HEAP) - { - printf("s"); - --msgLength; - break; - } - if (tree->gtFlags & GTF_IND_TGT_HEAP) - { - printf("h"); - --msgLength; - break; - } - if (tree->gtFlags & GTF_IND_INITCLASS) - { - printf("I"); - --msgLength; - break; - } - if (tree->gtFlags & GTF_IND_INVARIANT) - { - printf("#"); - --msgLength; - break; - } - if (tree->gtFlags & GTF_IND_NONFAULTING) - { - printf("n"); // print a n for non-faulting - --msgLength; - break; - } - if (tree->gtFlags & GTF_IND_NONNULL) - { - printf("@"); - --msgLength; - break; - } - } - if (tree->gtFlags & GTF_IND_VOLATILE) - { - printf("V"); - --msgLength; - break; - } - if (tree->gtFlags & GTF_IND_UNALIGNED) - { - printf("U"); - --msgLength; - break; - } - goto DASH; - - case GT_CALL: - if (tree->AsCall()->IsInlineCandidate()) - { - if (tree->AsCall()->IsGuardedDevirtualizationCandidate()) - { - printf("&"); - } - else - { - printf("I"); - } - --msgLength; - break; - } - else if (tree->AsCall()->IsGuardedDevirtualizationCandidate()) - { - printf("G"); - --msgLength; - break; - } - if (tree->AsCall()->gtCallMoreFlags & GTF_CALL_M_RETBUFFARG) - { - printf("S"); - --msgLength; - break; - } - if (tree->gtFlags & GTF_CALL_HOISTABLE) - { - printf("H"); - --msgLength; - break; - } - - goto DASH; - - case GT_MUL: -#if !defined(TARGET_64BIT) - case GT_MUL_LONG: -#endif - if (tree->gtFlags & GTF_MUL_64RSLT) - { - printf("L"); - --msgLength; - break; - } - goto DASH; - - case GT_LCL_FLD: - case GT_LCL_VAR: - case GT_LCL_ADDR: - case GT_STORE_LCL_FLD: - case GT_STORE_LCL_VAR: - if (tree->gtFlags & GTF_VAR_USEASG) - { - printf("U"); - --msgLength; - break; - } - if (tree->gtFlags & GTF_VAR_MULTIREG) - { - printf((tree->gtFlags & GTF_VAR_DEF) ? "M" : "m"); - --msgLength; - break; - } - if (tree->gtFlags & GTF_VAR_DEF) - { - printf("D"); - --msgLength; - break; - } - if (tree->gtFlags & GTF_VAR_CONTEXT) - { - printf("!"); - --msgLength; - break; - } - - goto DASH; - - case GT_EQ: - case GT_NE: - case GT_LT: - case GT_LE: - case GT_GE: - case GT_GT: - case GT_TEST_EQ: - case GT_TEST_NE: - case GT_SELECT: - if (tree->gtFlags & GTF_RELOP_NAN_UN) - { - printf("N"); - --msgLength; - break; - } - if (tree->gtFlags & GTF_RELOP_JMP_USED) - { - printf("J"); - --msgLength; - break; - } - goto DASH; - - case GT_CNS_INT: - if (tree->IsIconHandle()) - { - printf("H"); - --msgLength; - break; - } - goto DASH; - - default: - DASH: - printf("-"); - --msgLength; - break; - } - - /* Then print the general purpose flags */ - GenTreeFlags flags = tree->gtFlags; - - if (tree->OperIsBinary() || tree->OperIsMultiOp()) - { - genTreeOps oper = tree->OperGet(); - - // Check for GTF_ADDRMODE_NO_CSE flag on add/mul/shl Binary Operators - if ((oper == GT_ADD) || (oper == GT_MUL) || (oper == GT_LSH)) - { - if ((tree->gtFlags & GTF_ADDRMODE_NO_CSE) != 0) - { - flags |= GTF_DONT_CSE; // Force the GTF_ADDRMODE_NO_CSE flag to print out like GTF_DONT_CSE - } - } - } - else // !(tree->OperIsBinary() || tree->OperIsMultiOp()) - { - // the GTF_REVERSE flag only applies to binary operations (which some MultiOp nodes are). - flags &= ~GTF_REVERSE_OPS; - } - - msgLength -= GenTree::gtDispFlags(flags, tree->gtDebugFlags); - /* - printf("%c", (flags & GTF_ASG ) ? 'A' : '-'); - printf("%c", (flags & GTF_CALL ) ? 'C' : '-'); - printf("%c", (flags & GTF_EXCEPT ) ? 'X' : '-'); - printf("%c", (flags & GTF_GLOB_REF ) ? 'G' : '-'); - printf("%c", (flags & GTF_ORDER_SIDEEFF ) ? 'O' : '-'); - printf("%c", (flags & GTF_COLON_COND ) ? '?' : '-'); - printf("%c", (flags & GTF_DONT_CSE ) ? 'N' : // N is for No cse - (flags & GTF_MAKE_CSE ) ? 'H' : '-'); // H is for Hoist this expr - printf("%c", (flags & GTF_REVERSE_OPS ) ? 'R' : '-'); - printf("%c", (flags & GTF_UNSIGNED ) ? 'U' : - (flags & GTF_BOOLEAN ) ? 'B' : '-'); - printf("%c", (flags & GTF_SET_FLAGS ) ? 'S' : '-'); - printf("%c", ((flags & (GTF_SPILL | GTF_SPILLED)) == (GTF_SPILL | GTF_SPILLED)) ? '#' : ((flags & - GTF_SPILLED) ? 'z' : ((flags & GTF_SPILL) ? 'Z' : '-'))); - */ - } - - // If we're printing a node for LIR, we use the space normally associated with the message - // to display the node's temp name (if any) - const bool hasOperands = tree->OperandsBegin() != tree->OperandsEnd(); - if (isLIR) - { - assert(msg == nullptr); - - // If the tree does not have any operands, we do not display the indent stack. This gives us - // two additional characters for alignment. - if (!hasOperands) - { - msgLength += 1; - } - - if (tree->IsValue()) - { - const size_t bufLength = msgLength - 1; - msg = reinterpret_cast(_alloca(bufLength * sizeof(char))); - sprintf_s(const_cast(msg), bufLength, "%c%d = %s", tree->IsUnusedValue() ? 'u' : 't', tree->gtTreeID, - hasOperands ? "" : " "); - } - } - - /* print the msg associated with the node */ - - if (msg == nullptr) - { - msg = ""; - } - if (msgLength < 0) - { - msgLength = 0; - } - - printf(isLIR ? " %+*s" : " %-*s", msgLength, msg); - - /* Indent the node accordingly */ - if (!isLIR || hasOperands) - { - printIndent(indentStack); - } - - gtDispNodeName(tree); - - assert(tree == nullptr || tree->gtOper < GT_COUNT); - - if (tree) - { - /* print the type of the node */ - if (tree->gtOper != GT_CAST) - { - printf(" %-6s", varTypeName(tree->TypeGet())); - - if (varTypeIsStruct(tree->TypeGet())) - { - ClassLayout* layout = nullptr; - - if (tree->OperIs(GT_BLK, GT_STORE_BLK)) - { - layout = tree->AsBlk()->GetLayout(); - } - else if (tree->OperIs(GT_LCL_VAR, GT_STORE_LCL_VAR)) - { - LclVarDsc* varDsc = lvaGetDesc(tree->AsLclVar()); - - if (varTypeIsStruct(varDsc->TypeGet())) - { - layout = varDsc->GetLayout(); - } - } - else if (tree->OperIs(GT_LCL_FLD, GT_STORE_LCL_FLD)) - { - layout = tree->AsLclFld()->GetLayout(); - } - - if (layout != nullptr) - { - gtDispClassLayout(layout, tree->TypeGet()); - } - } - - if (tree->OperIs(GT_INDEX_ADDR, GT_ARR_ADDR)) - { - var_types elemType = - tree->OperIs(GT_INDEX_ADDR) ? tree->AsIndexAddr()->gtElemType : tree->AsArrAddr()->GetElemType(); - - CORINFO_CLASS_HANDLE elemClsHnd = tree->OperIs(GT_INDEX_ADDR) ? tree->AsIndexAddr()->gtStructElemClass - : tree->AsArrAddr()->GetElemClassHandle(); - - if (varTypeIsStruct(elemType) && (elemClsHnd != NO_CLASS_HANDLE)) - { - printf("%s[]", eeGetShortClassName(elemClsHnd)); - } - else - { - printf("%s[]", varTypeName(elemType)); - } - } - - if (tree->OperIsLocal()) - { - LclVarDsc* varDsc = lvaGetDesc(tree->AsLclVarCommon()); - if (varDsc->IsAddressExposed()) - { - printf("(AX)"); // Variable has address exposed. - } - if (varDsc->IsHiddenBufferStructArg()) - { - printf("(RB)"); // Variable is hidden return buffer - } - if (varDsc->lvUnusedStruct) - { - assert(varDsc->lvPromoted); - printf("(U)"); // Unused struct - } - else if (varDsc->lvPromoted) - { - if (varTypeIsPromotable(varDsc)) - { - printf("(P)"); // Promoted struct - } - else - { - // Promoted implicit by-refs can have this state during - // global morph while they are being rewritten - printf("(P?!)"); // Promoted struct - } - } - } - - if (tree->gtOper == GT_RUNTIMELOOKUP) - { -#ifdef TARGET_64BIT - printf(" 0x%llx", dspPtr(tree->AsRuntimeLookup()->gtHnd)); -#else - printf(" 0x%x", dspPtr(tree->AsRuntimeLookup()->gtHnd)); -#endif - - switch (tree->AsRuntimeLookup()->gtHndType) - { - case CORINFO_HANDLETYPE_CLASS: - printf(" class"); - break; - case CORINFO_HANDLETYPE_METHOD: - printf(" method"); - break; - case CORINFO_HANDLETYPE_FIELD: - printf(" field"); - break; - default: - printf(" unknown"); - break; - } - } - - if (tree->OperIs(GT_MDARR_LENGTH, GT_MDARR_LOWER_BOUND)) - { - printf(" (%u)", tree->AsMDArr()->Dim()); - } - } - - // for tracking down problems in reguse prediction or liveness tracking - - if (verbose && 0) - { - printf(" RR="); - dspRegMask(JitTls::GetCompiler()->codeGen->internalRegisters.GetAll(tree)); - printf("\n"); - } - } -} - -#if FEATURE_MULTIREG_RET -//---------------------------------------------------------------------------------- -// gtDispMultiRegCount: determine how many registers to print for a multi-reg node -// -// Arguments: -// tree - GenTree node whose registers we want to print -// -// Return Value: -// The number of registers to print -// -// Notes: -// This is not the same in all cases as GenTree::GetMultiRegCount(). -// In particular, for COPY or RELOAD it only returns the number of *valid* registers, -// and for CALL, it will return 0 if the ReturnTypeDesc hasn't yet been initialized. -// But we want to print all register positions. -// -unsigned Compiler::gtDispMultiRegCount(GenTree* tree) -{ - if (tree->IsCopyOrReload()) - { - // GetRegCount() will return only the number of valid regs for COPY or RELOAD, - // but we want to print all positions, so we get the reg count for op1. - return gtDispMultiRegCount(tree->gtGetOp1()); - } - else if (!tree->IsMultiRegNode()) - { - // We can wind up here because IsMultiRegNode() always returns true for COPY or RELOAD, - // even if its op1 is not multireg. - // Note that this method won't be called for non-register-producing nodes. - return 1; - } - else if (tree->OperIs(GT_CALL)) - { - unsigned regCount = tree->AsCall()->GetReturnTypeDesc()->TryGetReturnRegCount(); - // If it hasn't yet been initialized, we'd still like to see the registers printed. - if (regCount == 0) - { - regCount = MAX_RET_REG_COUNT; - } - return regCount; - } - else - { - return tree->GetMultiRegCount(this); - } -} -#endif // FEATURE_MULTIREG_RET - -//---------------------------------------------------------------------------------- -// gtDispRegVal: Print the register(s) defined by the given node -// -// Arguments: -// tree - GenTree node whose registers we want to print -// -void Compiler::gtDispRegVal(GenTree* tree) -{ - switch (tree->GetRegTag()) - { - // Don't display anything for the GT_REGTAG_NONE case; - // the absence of printed register values will imply this state. - - case GenTree::GT_REGTAG_REG: - printf(" REG %s", compRegVarName(tree->GetRegNum())); - break; - - default: - return; - } - -#if FEATURE_MULTIREG_RET - if (tree->IsMultiRegNode()) - { - // 0th reg is GetRegNum(), which is already printed above. - // Print the remaining regs of a multi-reg node. - unsigned regCount = gtDispMultiRegCount(tree); - - // For some nodes, e.g. COPY, RELOAD or CALL, we may not have valid regs for all positions. - for (unsigned i = 1; i < regCount; ++i) - { - regNumber reg = tree->GetRegByIndex(i); - printf(",%s", genIsValidReg(reg) ? compRegVarName(reg) : "NA"); - } - } -#endif -} - -// We usually/commonly don't expect to print anything longer than this string, -#define LONGEST_COMMON_LCL_VAR_DISPLAY "V99 PInvokeFrame" -#define LONGEST_COMMON_LCL_VAR_DISPLAY_LENGTH (sizeof(LONGEST_COMMON_LCL_VAR_DISPLAY)) -#define BUF_SIZE (LONGEST_COMMON_LCL_VAR_DISPLAY_LENGTH * 2) - -void Compiler::gtGetLclVarNameInfo(unsigned lclNum, const char** ilKindOut, const char** ilNameOut, unsigned* ilNumOut) -{ - const char* ilKind = nullptr; - const char* ilName = nullptr; - - unsigned ilNum = compMap2ILvarNum(lclNum); - - if (ilNum == (unsigned)ICorDebugInfo::RETBUF_ILNUM) - { - ilName = "RetBuf"; - } - else if (ilNum == (unsigned)ICorDebugInfo::VARARGS_HND_ILNUM) - { - ilName = "VarArgHandle"; - } - else if (ilNum == (unsigned)ICorDebugInfo::TYPECTXT_ILNUM) - { - ilName = "TypeCtx"; - } - else if (ilNum == (unsigned)ICorDebugInfo::UNKNOWN_ILNUM) - { - if (lclNumIsTrueCSE(lclNum)) - { - ilKind = "cse"; - ilNum = lclNum - optCSEstart; - } -#ifdef TARGET_ARM64 - else if (lclNum == lvaFfrRegister) - { - // We introduce this LclVar in lowering, hence special case the printing of - // it instead of handling it in "rationalizer" below. - ilName = "FFReg"; - } -#endif - else if (lclNum >= optCSEstart) - { - // Currently any new LclVar's introduced after the CSE phase - // are believed to be created by the "rationalizer" that is what is meant by the "rat" prefix. - ilKind = "rat"; - ilNum = lclNum - (optCSEstart + optCSEcount); - } - else - { - if (lclNum == info.compLvFrameListRoot) - { - ilName = "FramesRoot"; - } - else if (lclNum == lvaInlinedPInvokeFrameVar) - { - ilName = "PInvokeFrame"; - } - else if (lclNum == lvaGSSecurityCookie) - { - ilName = "GsCookie"; - } - else if (lclNum == lvaRetAddrVar) - { - ilName = "ReturnAddress"; - } -#if FEATURE_FIXED_OUT_ARGS - else if (lclNum == lvaOutgoingArgSpaceVar) - { - ilName = "OutArgs"; - } -#endif // FEATURE_FIXED_OUT_ARGS -#if defined(FEATURE_EH_WINDOWS_X86) - else if (lclNum == lvaShadowSPslotsVar) - { - ilName = "EHSlots"; - } -#endif // FEATURE_EH_WINDOWS_X86 -#ifdef JIT32_GCENCODER - else if (lclNum == lvaLocAllocSPvar) - { - ilName = "LocAllocSP"; - } -#endif // JIT32_GCENCODER - else if (lclNum == lvaPSPSym) - { - ilName = "PSPSym"; - } - else - { - ilKind = "tmp"; - if (compIsForInlining()) - { - ilNum = lclNum - impInlineInfo->InlinerCompiler->info.compLocalsCount; - } - else - { - ilNum = lclNum - info.compLocalsCount; - } - } - } - } - else if (lclNum < (compIsForInlining() ? impInlineInfo->InlinerCompiler->info.compArgsCount : info.compArgsCount)) - { - if (ilNum == 0 && !info.compIsStatic) - { - ilName = "this"; - } - else - { - ilKind = "arg"; - } - } - else - { - if (!lvaTable[lclNum].lvIsStructField) - { - ilKind = "loc"; - } - if (compIsForInlining()) - { - ilNum -= impInlineInfo->InlinerCompiler->info.compILargsCount; - } - else - { - ilNum -= info.compILargsCount; - } - } - - *ilKindOut = ilKind; - *ilNameOut = ilName; - *ilNumOut = ilNum; -} - -/*****************************************************************************/ -int Compiler::gtGetLclVarName(unsigned lclNum, char* buf, unsigned buf_remaining) -{ - char* bufp_next = buf; - unsigned charsPrinted = 0; - int sprintf_result; - - sprintf_result = sprintf_s(bufp_next, buf_remaining, "V%02u", lclNum); - - if (sprintf_result < 0) - { - return sprintf_result; - } - - charsPrinted += sprintf_result; - bufp_next += sprintf_result; - buf_remaining -= sprintf_result; - - const char* ilKind = nullptr; - const char* ilName = nullptr; - unsigned ilNum = 0; - - gtGetLclVarNameInfo(lclNum, &ilKind, &ilName, &ilNum); - - if (ilName != nullptr) - { - sprintf_result = sprintf_s(bufp_next, buf_remaining, " %s", ilName); - if (sprintf_result < 0) - { - return sprintf_result; - } - charsPrinted += sprintf_result; - bufp_next += sprintf_result; - buf_remaining -= sprintf_result; - } - else if (ilKind != nullptr) - { - sprintf_result = sprintf_s(bufp_next, buf_remaining, " %s%d", ilKind, ilNum); - if (sprintf_result < 0) - { - return sprintf_result; - } - charsPrinted += sprintf_result; - bufp_next += sprintf_result; - buf_remaining -= sprintf_result; - } - - assert(charsPrinted > 0); - assert(buf_remaining > 0); - - return (int)charsPrinted; -} - -/***************************************************************************** - * Get the local var name, and create a copy of the string that can be used in debug output. - */ -char* Compiler::gtGetLclVarName(unsigned lclNum) -{ - char buf[BUF_SIZE]; - int charsPrinted = gtGetLclVarName(lclNum, buf, ArrLen(buf)); - if (charsPrinted < 0) - { - return nullptr; - } - - char* retBuf = new (this, CMK_DebugOnly) char[charsPrinted + 1]; - strcpy_s(retBuf, charsPrinted + 1, buf); - return retBuf; -} - -/*****************************************************************************/ -void Compiler::gtDispLclVar(unsigned lclNum, bool padForBiggestDisp) -{ - char buf[BUF_SIZE]; - int charsPrinted = gtGetLclVarName(lclNum, buf, ArrLen(buf)); - - if (charsPrinted < 0) - { - return; - } - - printf("%s", buf); - - if (padForBiggestDisp && (charsPrinted < (int)LONGEST_COMMON_LCL_VAR_DISPLAY_LENGTH)) - { - printf("%*c", LONGEST_COMMON_LCL_VAR_DISPLAY_LENGTH - charsPrinted, ' '); - } -} - -//------------------------------------------------------------------------ -// gtDispLclVarStructType: Print size and type information about a struct or lclBlk local variable. -// -// Arguments: -// lclNum - The local var id. -// -void Compiler::gtDispLclVarStructType(unsigned lclNum) -{ - LclVarDsc* varDsc = lvaGetDesc(lclNum); - var_types type = varDsc->TypeGet(); - if (type == TYP_STRUCT) - { - ClassLayout* layout = varDsc->GetLayout(); - assert(layout != nullptr); - gtDispClassLayout(layout, type); - } -} - -#if defined(DEBUG) && defined(TARGET_ARM64) -static const char* InsCflagsToString(insCflags flags) -{ - const static char* s_table[16] = {"0", "v", "c", "cv", "z", "zv", "zc", "zcv", - "n", "nv", "nc", "ncv", "nz", "nzv", "nzc", "nzcv"}; - unsigned index = (unsigned)flags; - assert((0 <= index) && (index < ArrLen(s_table))); - return s_table[index]; -} -#endif - -//------------------------------------------------------------------------ -// gtDispSsaName: Display the SSA use/def for a given local. -// -// Arguments: -// lclNum - The local's number. -// ssaNum - The SSA number. -// isDef - Whether this is a def. -// -void Compiler::gtDispSsaName(unsigned lclNum, unsigned ssaNum, bool isDef) -{ - if (ssaNum == SsaConfig::RESERVED_SSA_NUM) - { - return; - } - - LclVarDsc* const lclDsc = lvaGetDesc(lclNum); - bool const isValid = lclDsc->IsValidSsaNum(ssaNum); - - if (isDef) - { - if (!isValid) - { - printf("?d:%d", ssaNum); - return; - } - - unsigned oldDefSsaNum = lclDsc->GetPerSsaData(ssaNum)->GetUseDefSsaNum(); - if (oldDefSsaNum != SsaConfig::RESERVED_SSA_NUM) - { - printf("ud:%d->%d", oldDefSsaNum, ssaNum); - return; - } - } - - printf("%s%s:%d", isValid ? "" : "?", isDef ? "d" : "u", ssaNum); -} - -//------------------------------------------------------------------------ -// gtDispClassLayout: Print size and type information about a layout. -// -// Arguments: -// layout - the layout; -// type - variable type, used to avoid printing size for SIMD nodes. -// -void Compiler::gtDispClassLayout(ClassLayout* layout, var_types type) -{ - assert(layout != nullptr); - if (layout->IsBlockLayout()) - { - printf("<%u>", layout->GetSize()); - } - else if (varTypeIsSIMD(type)) - { - printf("<%s>", layout->GetShortClassName()); - } - else - { - printf("<%s, %u>", layout->GetShortClassName(), layout->GetSize()); - } -} - -/*****************************************************************************/ -void Compiler::gtDispConst(GenTree* tree) -{ - assert(tree->OperIsConst()); - - switch (tree->gtOper) - { - case GT_CNS_INT: - if (tree->IsIconHandle(GTF_ICON_STR_HDL)) - { - printf(" 0x%X [ICON_STR_HDL]", dspPtr(tree->AsIntCon()->gtIconVal)); - } - else if (tree->IsIconHandle(GTF_ICON_OBJ_HDL)) - { - eePrintObjectDescription(" ", (CORINFO_OBJECT_HANDLE)tree->AsIntCon()->gtIconVal); - } - else - { - ssize_t iconVal = tree->AsIntCon()->gtIconVal; - ssize_t dspIconVal = tree->IsIconHandle() ? dspPtr(iconVal) : iconVal; - - if (tree->TypeGet() == TYP_REF) - { - if (iconVal == 0) - { - printf(" null"); - } - else - { - assert(doesMethodHaveFrozenObjects()); - printf(" 0x%llx", dspIconVal); - } - } - else if ((iconVal > -1000) && (iconVal < 1000)) - { - printf(" %ld", dspIconVal); - } -#ifdef TARGET_64BIT - else if ((iconVal & 0xFFFFFFFF00000000LL) != 0) - { - if (dspIconVal >= 0) - { - printf(" 0x%llx", dspIconVal); - } - else - { - printf(" -0x%llx", -dspIconVal); - } - } -#endif - else - { - if (dspIconVal >= 0) - { - printf(" 0x%X", dspIconVal); - } - else - { - printf(" -0x%X", -dspIconVal); - } - } - - if (tree->IsIconHandle()) - { - switch (tree->GetIconHandleFlag()) - { - case GTF_ICON_SCOPE_HDL: - printf(" scope"); - break; - case GTF_ICON_CLASS_HDL: - if (IsTargetAbi(CORINFO_NATIVEAOT_ABI) || opts.IsReadyToRun()) - { - printf(" class"); - } - else - { - printf(" class %s", eeGetClassName((CORINFO_CLASS_HANDLE)iconVal)); - } - break; - case GTF_ICON_METHOD_HDL: - if (IsTargetAbi(CORINFO_NATIVEAOT_ABI) || opts.IsReadyToRun()) - { - printf(" method"); - } - else - { - printf(" method %s", eeGetMethodFullName((CORINFO_METHOD_HANDLE)iconVal)); - } - break; - case GTF_ICON_FIELD_HDL: - if (IsTargetAbi(CORINFO_NATIVEAOT_ABI) || opts.IsReadyToRun()) - { - printf(" field"); - } - else - { - printf(" field %s", eeGetFieldName((CORINFO_FIELD_HANDLE)iconVal, true)); - } - break; - case GTF_ICON_STATIC_HDL: - printf(" static"); - break; - case GTF_ICON_OBJ_HDL: - case GTF_ICON_STR_HDL: - unreached(); // These cases are handled above - break; - case GTF_ICON_CONST_PTR: - printf(" const ptr"); - break; - case GTF_ICON_GLOBAL_PTR: - printf(" global ptr"); - break; - case GTF_ICON_VARG_HDL: - printf(" vararg"); - break; - case GTF_ICON_PINVKI_HDL: - printf(" pinvoke"); - break; - case GTF_ICON_TOKEN_HDL: - printf(" token"); - break; - case GTF_ICON_TLS_HDL: - printf(" tls"); - break; - case GTF_ICON_FTN_ADDR: - printf(" ftn"); - break; - case GTF_ICON_CIDMID_HDL: - printf(" cid/mid"); - break; - case GTF_ICON_BBC_PTR: - printf(" bbc"); - break; - case GTF_ICON_STATIC_BOX_PTR: - printf(" static box ptr"); - break; - case GTF_ICON_STATIC_ADDR_PTR: - printf(" static base addr cell"); - break; - case GTF_ICON_SECREL_OFFSET: - printf(" relative offset in section"); - break; - case GTF_ICON_TLSGD_OFFSET: - printf(" tls global dynamic offset"); - break; - default: - printf(" UNKNOWN"); - break; - } - } - -#ifdef FEATURE_SIMD - if ((tree->gtFlags & GTF_ICON_SIMD_COUNT) != 0) - { - printf(" vector element count"); - } -#endif - - if ((tree->IsReuseRegVal()) != 0) - { - printf(" reuse reg val"); - } - } - - if (tree->AsIntCon()->gtFieldSeq != nullptr) - { - FieldSeq* fieldSeq = tree->AsIntCon()->gtFieldSeq; - gtDispFieldSeq(fieldSeq, tree->AsIntCon()->IconValue() - fieldSeq->GetOffset()); - } - break; - - case GT_CNS_LNG: - printf(" 0x%016I64x", tree->AsLngCon()->gtLconVal); - break; - - case GT_CNS_DBL: - { - double dcon = tree->AsDblCon()->DconValue(); - if (FloatingPointUtils::isNegativeZero(dcon)) - { - printf(" -0.00000"); - } - else if (FloatingPointUtils::isNaN(dcon)) - { - uint64_t bits; - static_assert_no_msg(sizeof(bits) == sizeof(dcon)); - memcpy(&bits, &dcon, sizeof(dcon)); - printf(" %#.17g(0x%llx)\n", dcon, bits); - } - else - { - printf(" %#.17g", dcon); - } - break; - } - - case GT_CNS_STR: - printf(""); - break; - -#if defined(FEATURE_SIMD) - case GT_CNS_VEC: - { - GenTreeVecCon* vecCon = tree->AsVecCon(); - - switch (vecCon->TypeGet()) - { - case TYP_SIMD8: - { - printf("<0x%08x, 0x%08x>", vecCon->gtSimdVal.u32[0], vecCon->gtSimdVal.u32[1]); - break; - } - - case TYP_SIMD12: - { - printf("<0x%08x, 0x%08x, 0x%08x>", vecCon->gtSimdVal.u32[0], vecCon->gtSimdVal.u32[1], - vecCon->gtSimdVal.u32[2]); - break; - } - - case TYP_SIMD16: - { - printf("<0x%08x, 0x%08x, 0x%08x, 0x%08x>", vecCon->gtSimdVal.u32[0], vecCon->gtSimdVal.u32[1], - vecCon->gtSimdVal.u32[2], vecCon->gtSimdVal.u32[3]); - break; - } - -#if defined(TARGET_XARCH) - case TYP_SIMD32: - { - printf("<0x%016llx, 0x%016llx, 0x%016llx, 0x%016llx>", vecCon->gtSimdVal.u64[0], - vecCon->gtSimdVal.u64[1], vecCon->gtSimdVal.u64[2], vecCon->gtSimdVal.u64[3]); - break; - } - - case TYP_SIMD64: - { - printf("<0x%016llx, 0x%016llx, 0x%016llx, 0x%016llx, 0x%016llx, 0x%016llx, 0x%016llx, 0x%016llx>", - vecCon->gtSimdVal.u64[0], vecCon->gtSimdVal.u64[1], vecCon->gtSimdVal.u64[2], - vecCon->gtSimdVal.u64[3], vecCon->gtSimdVal.u64[4], vecCon->gtSimdVal.u64[5], - vecCon->gtSimdVal.u64[6], vecCon->gtSimdVal.u64[7]); - break; - } - -#endif // TARGET_XARCH - - default: - { - unreached(); - } - } - - break; - } -#endif // FEATURE_SIMD - -#if defined(FEATURE_MASKED_HW_INTRINSICS) - case GT_CNS_MSK: - { - GenTreeMskCon* mskCon = tree->AsMskCon(); - printf("<0x%08x, 0x%08x>", mskCon->gtSimdMaskVal.u32[0], mskCon->gtSimdMaskVal.u32[1]); - break; - } -#endif // FEATURE_MASKED_HW_INTRINSICS - - default: - assert(!"unexpected constant node"); - } -} - -//------------------------------------------------------------------------ -// gtDispFieldSeq: Print out the fields in this field sequence. -// -// Arguments: -// fieldSeq - The field sequence -// offset - Offset of the (implicit) struct fields in the sequence -// -void Compiler::gtDispFieldSeq(FieldSeq* fieldSeq, ssize_t offset) -{ - if (fieldSeq == nullptr) - { - return; - } - - char buffer[128]; - printf(" Fseq[%s", eeGetFieldName(fieldSeq->GetFieldHandle(), false, buffer, sizeof(buffer))); - if (offset != 0) - { - printf(", %zd", offset); - } - printf("]"); -} - -//------------------------------------------------------------------------ -// gtDispLeaf: Print a single leaf node to jitstdout. -// -// Arguments: -// tree - the tree to be printed -// indentStack - the specification for the current level of indentation & arcs -// -// Return Value: -// None. -// -// Notes: -// 'indentStack' may be null, in which case no indentation or arcs are printed - -void Compiler::gtDispLeaf(GenTree* tree, IndentStack* indentStack) -{ - if (tree->OperIsConst()) - { - gtDispConst(tree); - return; - } - - char buffer[256]; - - switch (tree->gtOper) - { - case GT_PHI_ARG: - case GT_LCL_VAR: - case GT_LCL_FLD: - case GT_LCL_ADDR: - gtDispLocal(tree->AsLclVarCommon(), indentStack); - break; - - case GT_JMP: - { - printf(" %s", eeGetMethodFullName((CORINFO_METHOD_HANDLE)tree->AsVal()->gtVal1, true, true, buffer, - sizeof(buffer))); - } - break; - - case GT_LABEL: - break; - - case GT_FTN_ADDR: - { - printf(" %s\n", eeGetMethodFullName((CORINFO_METHOD_HANDLE)tree->AsFptrVal()->gtFptrMethod, true, true, - buffer, sizeof(buffer))); - } - break; - -#if defined(FEATURE_EH_WINDOWS_X86) - case GT_END_LFIN: - printf(" endNstLvl=%d", tree->AsVal()->gtVal1); - break; -#endif // FEATURE_EH_WINDOWS_X86 - - // Vanilla leaves. No qualifying information available. So do nothing - - case GT_NOP: - case GT_NO_OP: - case GT_START_NONGC: - case GT_START_PREEMPTGC: - case GT_PROF_HOOK: - case GT_CATCH_ARG: - case GT_MEMORYBARRIER: - case GT_PINVOKE_PROLOG: - case GT_JMPTABLE: - case GT_SWIFT_ERROR: - break; - - case GT_RET_EXPR: - { - GenTreeCall* inlineCand = tree->AsRetExpr()->gtInlineCandidate; - printf("(for "); - printTreeID(inlineCand); - printf(")"); - - if (tree->AsRetExpr()->gtSubstExpr != nullptr) - { - printf(" -> "); - printTreeID(tree->AsRetExpr()->gtSubstExpr); - } - } - break; - - case GT_PHYSREG: - printf(" %s", getRegName(tree->AsPhysReg()->gtSrcReg)); - break; - - case GT_IL_OFFSET: - printf(" "); - tree->AsILOffset()->gtStmtDI.Dump(true); - break; - - case GT_JCC: - case GT_SETCC: - printf(" cond=%s", tree->AsCC()->gtCondition.Name()); - break; - case GT_JCMP: - case GT_JTEST: - printf(" cond=%s", tree->AsOpCC()->gtCondition.Name()); - break; - - default: - assert(!"don't know how to display tree leaf node"); - } -} - -//------------------------------------------------------------------------ -// gtDispLocal: Print description of a local node to jitstdout. -// -// Prints the information common to all local nodes. Does not print children. -// -// Arguments: -// tree - the local tree -// indentStack - the specification for the current level of indentation & arcs -// -void Compiler::gtDispLocal(GenTreeLclVarCommon* tree, IndentStack* indentStack) -{ - printf(" "); - const unsigned varNum = tree->GetLclNum(); - const LclVarDsc* varDsc = lvaGetDesc(varNum); - const bool isDef = (tree->gtFlags & GTF_VAR_DEF) != 0; - const bool isLclFld = tree->OperIsLocalField(); - - gtDispLclVar(varNum); - gtDispSsaName(varNum, tree->GetSsaNum(), isDef); - - if (isLclFld) - { - printf("[+%u]", tree->AsLclFld()->GetLclOffs()); - } - - if (varDsc->lvRegister) - { - printf(" "); - varDsc->PrintVarReg(); - } - else if (tree->InReg()) - { - printf(" %s", compRegVarName(tree->GetRegNum())); - } - - if (varDsc->lvPromoted) - { - if (!varTypeIsPromotable(varDsc) && !varDsc->lvUnusedStruct) - { - // Promoted implicit byrefs can get in this state while they are being rewritten - // in global morph. - } - else - { - for (unsigned index = 0; index < varDsc->lvFieldCnt; index++) - { - unsigned fieldLclNum = varDsc->lvFieldLclStart + index; - LclVarDsc* fieldVarDsc = lvaGetDesc(fieldLclNum); - - printf("\n"); - printf(" "); - printIndent(indentStack); - printf(" %-6s %s -> ", varTypeName(fieldVarDsc->TypeGet()), fieldVarDsc->lvReason); - gtDispLclVar(fieldLclNum); - gtDispSsaName(fieldLclNum, tree->GetSsaNum(this, index), isDef); - - if (fieldVarDsc->lvRegister) - { - printf(" "); - fieldVarDsc->PrintVarReg(); - } - - if (fieldVarDsc->lvTracked && fgLocalVarLivenessDone && tree->IsLastUse(index)) - { - printf(" (last use)"); - } - } - } - } - else // a normal not-promoted lclvar - { - if ((varDsc->lvTracked || varDsc->lvTrackedWithoutIndex) && fgLocalVarLivenessDone && - ((tree->gtFlags & GTF_VAR_DEATH) != 0)) - { - printf(" (last use)"); - } - } -} - -//------------------------------------------------------------------------ -// gtDispChild: Print a child node to jitstdout. -// -// Arguments: -// tree - the tree to be printed -// indentStack - the specification for the current level of indentation & arcs -// arcType - the type of arc to use for this child -// msg - a contextual method (i.e. from the parent) to print -// topOnly - a boolean indicating whether to print the children, or just the top node -// -// Return Value: -// None. -// -// Notes: -// 'indentStack' may be null, in which case no indentation or arcs are printed -// 'msg' has a default value of null -// 'topOnly' is an optional argument that defaults to false - -void Compiler::gtDispChild(GenTree* child, - IndentStack* indentStack, - IndentInfo arcType, - _In_opt_ const char* msg, /* = nullptr */ - bool topOnly) /* = false */ -{ - indentStack->Push(arcType); - gtDispTree(child, indentStack, msg, topOnly); - indentStack->Pop(); -} - -/*****************************************************************************/ - -void Compiler::gtDispTree(GenTree* tree, - IndentStack* indentStack, /* = nullptr */ - _In_ _In_opt_z_ const char* msg, /* = nullptr */ - bool topOnly, /* = false */ - bool isLIR) /* = false */ -{ - if (tree == nullptr) - { - printf(" [%08X] \n", tree); - printf(""); // null string means flush - return; - } - - if (indentStack == nullptr) - { - indentStack = new (this, CMK_DebugOnly) IndentStack(this); - } - - if (IsUninitialized(tree)) - { - /* Value used to initialize nodes */ - printf("Uninitialized tree node!\n"); - return; - } - - if (tree->gtOper >= GT_COUNT) - { - gtDispNode(tree, indentStack, msg, isLIR); - printf("Bogus operator!\n"); - return; - } - - // Determine what kind of arc to propagate. - IndentInfo myArc = IINone; - IndentInfo lowerArc = IINone; - if (indentStack->Depth() > 0) - { - myArc = indentStack->Pop(); - switch (myArc) - { - case IIArcBottom: - indentStack->Push(IIArc); - lowerArc = IINone; - break; - case IIArc: - indentStack->Push(IIArc); - lowerArc = IIArc; - break; - case IIArcTop: - indentStack->Push(IINone); - lowerArc = IIArc; - break; - case IINone: - indentStack->Push(IINone); - lowerArc = IINone; - break; - default: - unreached(); - break; - } - } - - /* Is it a 'simple' unary/binary operator? */ - - const char* childMsg = nullptr; - - if (tree->OperIsSimple()) - { - // Now, get the right type of arc for this node - if (myArc != IINone) - { - indentStack->Pop(); - indentStack->Push(myArc); - } - - gtDispNode(tree, indentStack, msg, isLIR); - - // Propagate lowerArc to the lower children. - if (indentStack->Depth() > 0) - { - (void)indentStack->Pop(); - indentStack->Push(lowerArc); - } - - if (tree->gtOper == GT_CAST) - { - /* Format a message that explains the effect of this GT_CAST */ - - var_types fromType = genActualType(tree->AsCast()->CastOp()->TypeGet()); - var_types toType = tree->CastToType(); - var_types finalType = tree->TypeGet(); - - /* if GTF_UNSIGNED is set then force fromType to an unsigned type */ - if (tree->gtFlags & GTF_UNSIGNED) - { - fromType = varTypeToUnsigned(fromType); - } - - if (finalType != toType) - { - printf(" %s <-", varTypeName(finalType)); - } - - printf(" %s <- %s", varTypeName(toType), varTypeName(fromType)); - } - - if (tree->OperIsLocalStore()) // Local stores used to be leaf nodes. - { - gtDispLocal(tree->AsLclVarCommon(), indentStack); - } - else if (tree->OperIsBlkOp()) - { - if (tree->OperIsCopyBlkOp()) - { - printf(" (copy)"); - } - else if (tree->OperIsInitBlkOp()) - { - printf(" (init)"); - } - if (tree->OperIsStoreBlk() && (tree->AsBlk()->gtBlkOpKind != GenTreeBlk::BlkOpKindInvalid)) - { - switch (tree->AsBlk()->gtBlkOpKind) - { - case GenTreeBlk::BlkOpKindCpObjUnroll: - printf(" (CpObjUnroll)"); - break; -#ifdef TARGET_XARCH - case GenTreeBlk::BlkOpKindCpObjRepInstr: - printf(" (CpObjRepInstr)"); - break; - - case GenTreeBlk::BlkOpKindRepInstr: - printf(" (RepInstr)"); - break; -#endif - case GenTreeBlk::BlkOpKindUnroll: - printf(" (Unroll)"); - break; - - case GenTreeBlk::BlkOpKindUnrollMemmove: - printf(" (Memmove)"); - break; - - case GenTreeBlk::BlkOpKindLoop: - printf(" (Loop)"); - break; - - default: - unreached(); - } - } - } -#if FEATURE_PUT_STRUCT_ARG_STK - else if (tree->OperGet() == GT_PUTARG_STK) - { - const GenTreePutArgStk* putArg = tree->AsPutArgStk(); - printf(" (%d stackByteSize), (%d byteOffset)", putArg->GetStackByteSize(), putArg->getArgOffset()); - if (putArg->gtPutArgStkKind != GenTreePutArgStk::Kind::Invalid) - { - switch (putArg->gtPutArgStkKind) - { - case GenTreePutArgStk::Kind::RepInstr: - printf(" (RepInstr)"); - break; - case GenTreePutArgStk::Kind::PartialRepInstr: - printf(" (PartialRepInstr)"); - break; - case GenTreePutArgStk::Kind::Unroll: - printf(" (Unroll)"); - break; - case GenTreePutArgStk::Kind::Push: - printf(" (Push)"); - break; - default: - unreached(); - } - } - } -#if FEATURE_ARG_SPLIT - else if (tree->OperGet() == GT_PUTARG_SPLIT) - { - const GenTreePutArgSplit* putArg = tree->AsPutArgSplit(); - printf(" (%d stackByteSize), (%d numRegs)", putArg->GetStackByteSize(), putArg->gtNumRegs); - } -#endif // FEATURE_ARG_SPLIT -#endif // FEATURE_PUT_STRUCT_ARG_STK - - if (tree->OperIs(GT_FIELD_ADDR)) - { - auto disp = [&]() { - char buffer[256]; - printf(" %s", eeGetFieldName(tree->AsFieldAddr()->gtFldHnd, true, buffer, sizeof(buffer))); - }; - disp(); - } - - if (tree->gtOper == GT_INTRINSIC) - { - GenTreeIntrinsic* intrinsic = tree->AsIntrinsic(); - - switch (intrinsic->gtIntrinsicName) - { - case NI_System_Math_Abs: - printf(" abs"); - break; - case NI_System_Math_Acos: - printf(" acos"); - break; - case NI_System_Math_Acosh: - printf(" acosh"); - break; - case NI_System_Math_Asin: - printf(" asin"); - break; - case NI_System_Math_Asinh: - printf(" asinh"); - break; - case NI_System_Math_Atan: - printf(" atan"); - break; - case NI_System_Math_Atanh: - printf(" atanh"); - break; - case NI_System_Math_Atan2: - printf(" atan2"); - break; - case NI_System_Math_Cbrt: - printf(" cbrt"); - break; - case NI_System_Math_Ceiling: - printf(" ceiling"); - break; - case NI_System_Math_Cos: - printf(" cos"); - break; - case NI_System_Math_Cosh: - printf(" cosh"); - break; - case NI_System_Math_Exp: - printf(" exp"); - break; - case NI_System_Math_Floor: - printf(" floor"); - break; - case NI_System_Math_FusedMultiplyAdd: - printf(" fma"); - break; - case NI_System_Math_ILogB: - printf(" ilogb"); - break; - case NI_System_Math_Log: - printf(" log"); - break; - case NI_System_Math_Log2: - printf(" log2"); - break; - case NI_System_Math_Log10: - printf(" log10"); - break; - case NI_System_Math_Max: - printf(" max"); - break; - case NI_System_Math_MaxMagnitude: - printf(" maxMagnitude"); - break; - case NI_System_Math_MaxMagnitudeNumber: - printf(" maxMagnitudeNumber"); - break; - case NI_System_Math_MaxNumber: - printf(" maxNumber"); - break; - case NI_System_Math_Min: - printf(" min"); - break; - case NI_System_Math_MinMagnitude: - printf(" minMagnitude"); - break; - case NI_System_Math_MinMagnitudeNumber: - printf(" minMagnitudeNumber"); - break; - case NI_System_Math_MinNumber: - printf(" minNumber"); - break; - case NI_System_Math_Pow: - printf(" pow"); - break; - case NI_System_Math_Round: - printf(" round"); - break; - case NI_System_Math_Sin: - printf(" sin"); - break; - case NI_System_Math_Sinh: - printf(" sinh"); - break; - case NI_System_Math_Sqrt: - printf(" sqrt"); - break; - case NI_System_Math_Tan: - printf(" tan"); - break; - case NI_System_Math_Tanh: - printf(" tanh"); - break; - case NI_System_Math_Truncate: - printf(" truncate"); - break; - case NI_System_Object_GetType: - printf(" objGetType"); - break; - case NI_System_Runtime_CompilerServices_RuntimeHelpers_IsKnownConstant: - printf(" isKnownConst"); - break; -#if defined(FEATURE_SIMD) - case NI_SIMD_UpperRestore: - printf(" simdUpperRestore"); - break; - case NI_SIMD_UpperSave: - printf(" simdUpperSave"); - break; -#endif // FEATURE_SIMD - default: - printf("Unknown intrinsic: "); - printTreeID(tree); - break; - } - } - else if (tree->OperIs(GT_SELECTCC)) - { - printf(" cond=%s", tree->AsOpCC()->gtCondition.Name()); - } -#ifdef TARGET_ARM64 - else if (tree->OperIs(GT_SELECT_INCCC, GT_SELECT_INVCC, GT_SELECT_NEGCC)) - { - printf(" cond=%s", tree->AsOpCC()->gtCondition.Name()); - } - else if (tree->OperIs(GT_CCMP)) - { - printf(" cond=%s flags=%s", tree->AsCCMP()->gtCondition.Name(), - InsCflagsToString(tree->AsCCMP()->gtFlagsVal)); - } -#endif - - gtDispCommonEndLine(tree); - - if (!topOnly) - { - if (tree->AsOp()->gtOp1 != nullptr) - { - // Label the child of the GT_COLON operator - // op1 is the else part - if (tree->gtOper == GT_COLON) - { - childMsg = "else"; - } - else if (tree->gtOper == GT_QMARK) - { - childMsg = " if"; - } - gtDispChild(tree->AsOp()->gtOp1, indentStack, - (tree->gtGetOp2IfPresent() == nullptr) ? IIArcBottom : IIArc, childMsg, topOnly); - } - - if (tree->gtGetOp2IfPresent()) - { - // Label the childMsgs of the GT_COLON operator - // op2 is the then part - - if (tree->gtOper == GT_COLON) - { - childMsg = "then"; - } - gtDispChild(tree->AsOp()->gtOp2, indentStack, IIArcBottom, childMsg, topOnly); - } - } - - return; - } - - // Now, get the right type of arc for this node - if (myArc != IINone) - { - indentStack->Pop(); - indentStack->Push(myArc); - } - gtDispNode(tree, indentStack, msg, isLIR); - - // Propagate lowerArc to the lower children. - if (indentStack->Depth() > 0) - { - (void)indentStack->Pop(); - indentStack->Push(lowerArc); - } - - // See what kind of a special operator we have here, and handle its special children. - - switch (tree->gtOper) - { - case GT_FIELD_LIST: - gtDispCommonEndLine(tree); - - if (!topOnly) - { - for (GenTreeFieldList::Use& use : tree->AsFieldList()->Uses()) - { - char offset[32]; - sprintf_s(offset, sizeof(offset), "ofs %u", use.GetOffset()); - gtDispChild(use.GetNode(), indentStack, (use.GetNext() == nullptr) ? IIArcBottom : IIArc, offset); - } - } - break; - - case GT_PHI: - gtDispCommonEndLine(tree); - - if (!topOnly) - { - for (GenTreePhi::Use& use : tree->AsPhi()->Uses()) - { - char block[32]; - sprintf_s(block, sizeof(block), "pred " FMT_BB, use.GetNode()->AsPhiArg()->gtPredBB->bbNum); - gtDispChild(use.GetNode(), indentStack, (use.GetNext() == nullptr) ? IIArcBottom : IIArc, block); - } - } - break; - - case GT_CALL: - { - GenTreeCall* call = tree->AsCall(); - GenTree* lastChild = nullptr; - call->VisitOperands([&lastChild](GenTree* operand) -> GenTree::VisitResult { - lastChild = operand; - return GenTree::VisitResult::Continue; - }); - - if (call->gtCallType != CT_INDIRECT) - { - auto disp = [&]() { - char buffer[256]; - printf(" %s", eeGetMethodFullName(call->gtCallMethHnd, true, true, buffer, sizeof(buffer))); - }; - disp(); - } - - if ((call->gtFlags & GTF_CALL_UNMANAGED) && (call->gtCallMoreFlags & GTF_CALL_M_FRAME_VAR_DEATH)) - { - printf(" (FramesRoot last use)"); - } - - if ((call->gtFlags & GTF_CALL_INLINE_CANDIDATE) != 0) - { - InlineCandidateInfo* inlineInfo; - if (call->IsGuardedDevirtualizationCandidate()) - { - inlineInfo = call->GetGDVCandidateInfo(0); - } - else - { - inlineInfo = call->GetSingleInlineCandidateInfo(); - } - - if ((inlineInfo != nullptr) && (inlineInfo->exactContextHandle != nullptr)) - { - printf(" (exactContextHandle=0x%p)", dspPtr(inlineInfo->exactContextHandle)); - } - } - - gtDispCommonEndLine(tree); - - if (!topOnly) - { - char buf[64]; - - gtDispArgList(call, lastChild, indentStack); - - if (call->gtCallType == CT_INDIRECT) - { - gtDispChild(call->gtCallAddr, indentStack, (call->gtCallAddr == lastChild) ? IIArcBottom : IIArc, - "calli tgt", topOnly); - } - - if (call->gtControlExpr != nullptr) - { - gtDispChild(call->gtControlExpr, indentStack, - (call->gtControlExpr == lastChild) ? IIArcBottom : IIArc, "control expr", topOnly); - } - - for (CallArg& arg : call->gtArgs.LateArgs()) - { - IndentInfo arcType = (arg.GetLateNext() == nullptr) ? IIArcBottom : IIArc; - gtGetLateArgMsg(call, &arg, buf, sizeof(buf)); - gtDispChild(arg.GetLateNode(), indentStack, arcType, buf, topOnly); - } - } - } - break; - -#if defined(FEATURE_HW_INTRINSICS) - case GT_HWINTRINSIC: - if (tree->OperIs(GT_HWINTRINSIC)) - { - printf(" %s %s", - tree->AsHWIntrinsic()->GetSimdBaseType() == TYP_UNKNOWN - ? "" - : varTypeName(tree->AsHWIntrinsic()->GetSimdBaseType()), - HWIntrinsicInfo::lookupName(tree->AsHWIntrinsic()->GetHWIntrinsicId())); - } - - gtDispCommonEndLine(tree); - - if (!topOnly) - { - size_t index = 0; - size_t count = tree->AsMultiOp()->GetOperandCount(); - for (GenTree* operand : tree->AsMultiOp()->Operands()) - { - gtDispChild(operand, indentStack, ++index < count ? IIArc : IIArcBottom, nullptr, topOnly); - } - } - break; -#endif // defined(FEATURE_HW_INTRINSICS) - - case GT_ARR_ELEM: - gtDispCommonEndLine(tree); - - if (!topOnly) - { - gtDispChild(tree->AsArrElem()->gtArrObj, indentStack, IIArc, nullptr, topOnly); - - unsigned dim; - for (dim = 0; dim < tree->AsArrElem()->gtArrRank; dim++) - { - IndentInfo arcType = ((dim + 1) == tree->AsArrElem()->gtArrRank) ? IIArcBottom : IIArc; - gtDispChild(tree->AsArrElem()->gtArrInds[dim], indentStack, arcType, nullptr, topOnly); - } - } - break; - - case GT_CMPXCHG: - gtDispCommonEndLine(tree); - - if (!topOnly) - { - gtDispChild(tree->AsCmpXchg()->Addr(), indentStack, IIArc, nullptr, topOnly); - gtDispChild(tree->AsCmpXchg()->Data(), indentStack, IIArc, nullptr, topOnly); - gtDispChild(tree->AsCmpXchg()->Comparand(), indentStack, IIArcBottom, nullptr, topOnly); - } - break; - - case GT_SELECT: - gtDispCommonEndLine(tree); - - if (!topOnly) - { - gtDispChild(tree->AsConditional()->gtCond, indentStack, IIArc, childMsg, topOnly); - gtDispChild(tree->AsConditional()->gtOp1, indentStack, IIArc, childMsg, topOnly); - gtDispChild(tree->AsConditional()->gtOp2, indentStack, IIArcBottom, childMsg, topOnly); - } - break; - - default: - if (tree->OperIsLeaf()) - { - gtDispLeaf(tree, indentStack); - gtDispCommonEndLine(tree); - } - else - { - printf(" :"); - printf(""); // null string means flush - } - break; - } -} - -//------------------------------------------------------------------------ -// gtGetWellKnownArgNameForArgMsg: Get a short descriptor of a well-known arg kind. -// -const char* Compiler::gtGetWellKnownArgNameForArgMsg(WellKnownArg arg) -{ - switch (arg) - { - case WellKnownArg::ThisPointer: - return "this"; - case WellKnownArg::VarArgsCookie: - return "va cookie"; - case WellKnownArg::InstParam: - return "gctx"; - case WellKnownArg::RetBuffer: - return "retbuf"; - case WellKnownArg::PInvokeFrame: - return "pinv frame"; - case WellKnownArg::WrapperDelegateCell: - return "wrap cell"; - case WellKnownArg::ShiftLow: - return "shift low"; - case WellKnownArg::ShiftHigh: - return "shift high"; - case WellKnownArg::VirtualStubCell: - return "vsd cell"; - case WellKnownArg::PInvokeCookie: - return "pinv cookie"; - case WellKnownArg::PInvokeTarget: - return "pinv tgt"; - case WellKnownArg::R2RIndirectionCell: - return "r2r cell"; - case WellKnownArg::ValidateIndirectCallTarget: - case WellKnownArg::DispatchIndirectCallTarget: - return "cfg tgt"; - case WellKnownArg::SwiftError: - return "swift error"; - case WellKnownArg::SwiftSelf: - return "swift self"; - case WellKnownArg::X86TailCallSpecialArg: - return "tail call"; - default: - return nullptr; - } -} - -//------------------------------------------------------------------------ -// gtPrintArgPrefix: Print a description of an argument into the specified buffer. -// -// Remarks: -// For well-known arguments this prints a human-readable description. -// Otherwise it prints e.g. "arg3". -// -void Compiler::gtPrintArgPrefix(GenTreeCall* call, CallArg* arg, char** bufp, unsigned* bufLength) -{ - int prefLen; - const char* wellKnownName = gtGetWellKnownArgNameForArgMsg(arg->GetWellKnownArg()); - if (wellKnownName != nullptr) - { - prefLen = sprintf_s(*bufp, *bufLength, "%s", wellKnownName); - } - else - { - unsigned argNum = call->gtArgs.GetIndex(arg); - prefLen = sprintf_s(*bufp, *bufLength, "arg%u", argNum); - } - assert(prefLen != -1); - *bufp += prefLen; - *bufLength -= (unsigned)prefLen; -} - -//------------------------------------------------------------------------ -// gtGetArgMsg: Construct a message about the given argument -// -// Arguments: -// call - The call for which 'arg' is an argument -// arg - The argument for which a message should be constructed -// bufp - A pointer to the buffer into which the message is written -// bufLength - The length of the buffer pointed to by bufp -// -// Return Value: -// No return value, but bufp is written. -// -void Compiler::gtGetArgMsg(GenTreeCall* call, CallArg* arg, char* bufp, unsigned bufLength) -{ - gtPrintArgPrefix(call, arg, &bufp, &bufLength); - - if (arg->GetLateNode() != nullptr) - { - sprintf_s(bufp, bufLength, " setup"); - } - else if (call->gtArgs.IsAbiInformationDetermined()) - { -#ifdef TARGET_ARM - if (arg->AbiInfo.IsSplit()) - { - regNumber firstReg = arg->AbiInfo.GetRegNum(); - if (arg->AbiInfo.NumRegs == 1) - { - sprintf_s(bufp, bufLength, " %s out+%02x", compRegVarName(firstReg), arg->AbiInfo.ByteOffset); - } - else - { - regNumber lastReg = REG_STK; - char separator = (arg->AbiInfo.NumRegs == 2) ? ',' : '-'; - if (arg->AbiInfo.IsHfaRegArg()) - { - unsigned lastRegNum = genMapFloatRegNumToRegArgNum(firstReg) + arg->AbiInfo.NumRegs - 1; - lastReg = genMapFloatRegArgNumToRegNum(lastRegNum); - } - else - { - unsigned lastRegNum = - genMapIntRegNumToRegArgNum(firstReg, call->GetUnmanagedCallConv()) + arg->AbiInfo.NumRegs - 1; - lastReg = genMapIntRegArgNumToRegNum(lastRegNum, call->GetUnmanagedCallConv()); - } - sprintf_s(bufp, bufLength, " %s%c%s out+%02x", compRegVarName(firstReg), separator, - compRegVarName(lastReg), arg->AbiInfo.ByteOffset); - } - - return; - } -#endif // TARGET_ARM -#if FEATURE_FIXED_OUT_ARGS - sprintf_s(bufp, bufLength, " out+%02x", arg->AbiInfo.ByteOffset); -#else - sprintf_s(bufp, bufLength, " on STK"); -#endif - } -} - -//------------------------------------------------------------------------ -// gtGetLateArgMsg: Construct a message about the given argument -// -// Arguments: -// call - The call for which 'arg' is an argument -// arg - The argument for which a message should be constructed -// bufp - A pointer to the buffer into which the message is written -// bufLength - The length of the buffer pointed to by bufp -// -// Return Value: -// No return value, but bufp is written. - -void Compiler::gtGetLateArgMsg(GenTreeCall* call, CallArg* arg, char* bufp, unsigned bufLength) -{ - assert(arg->GetLateNode() != nullptr); - regNumber argReg = arg->AbiInfo.GetRegNum(); - - gtPrintArgPrefix(call, arg, &bufp, &bufLength); - -#if FEATURE_FIXED_OUT_ARGS - if (argReg == REG_STK) - { - sprintf_s(bufp, bufLength, " in out+%02x", arg->AbiInfo.ByteOffset); - } - else -#endif - { -#ifdef TARGET_ARM - if (arg->AbiInfo.IsSplit()) - { - regNumber firstReg = arg->AbiInfo.GetRegNum(); - if (arg->AbiInfo.NumRegs == 1) - { - sprintf_s(bufp, bufLength, " %s out+%02x", compRegVarName(firstReg), arg->AbiInfo.ByteOffset); - } - else - { - regNumber lastReg = REG_STK; - char separator = (arg->AbiInfo.NumRegs == 2) ? ',' : '-'; - if (arg->AbiInfo.IsHfaRegArg()) - { - unsigned lastRegNum = genMapFloatRegNumToRegArgNum(firstReg) + arg->AbiInfo.NumRegs - 1; - lastReg = genMapFloatRegArgNumToRegNum(lastRegNum); - } - else - { - unsigned lastRegNum = - genMapIntRegNumToRegArgNum(firstReg, call->GetUnmanagedCallConv()) + arg->AbiInfo.NumRegs - 1; - lastReg = genMapIntRegArgNumToRegNum(lastRegNum, call->GetUnmanagedCallConv()); - } - sprintf_s(bufp, bufLength, " %s%c%s out+%02x", compRegVarName(firstReg), separator, - compRegVarName(lastReg), arg->AbiInfo.ByteOffset); - } - - return; - } -#endif // TARGET_ARM -#if FEATURE_MULTIREG_ARGS - if (arg->AbiInfo.NumRegs >= 2) - { - char separator = (arg->AbiInfo.NumRegs == 2) ? ',' : '-'; - sprintf_s(bufp, bufLength, " %s%c%s", compRegVarName(argReg), separator, - compRegVarName(arg->AbiInfo.GetRegNum(arg->AbiInfo.NumRegs - 1))); - } - else -#endif - { - sprintf_s(bufp, bufLength, " in %s", compRegVarName(argReg)); - } - } -} - -//------------------------------------------------------------------------ -// gtDispArgList: Dump the tree for a call arg list -// -// Arguments: -// call - the call to dump arguments for -// lastCallOperand - the call's last operand (to determine the arc types) -// indentStack - the specification for the current level of indentation & arcs -// -// Return Value: -// None. -// -void Compiler::gtDispArgList(GenTreeCall* call, GenTree* lastCallOperand, IndentStack* indentStack) -{ - for (CallArg& arg : call->gtArgs.EarlyArgs()) - { - char buf[256]; - gtGetArgMsg(call, &arg, buf, sizeof(buf)); - gtDispChild(arg.GetEarlyNode(), indentStack, (arg.GetEarlyNode() == lastCallOperand) ? IIArcBottom : IIArc, buf, - false); - } -} - -// gtDispStmt: Print a statement to jitstdout. -// -// Arguments: -// stmt - the statement to be printed; -// msg - an additional message to print before the statement. -// -void Compiler::gtDispStmt(Statement* stmt, const char* msg /* = nullptr */) -{ - if (msg != nullptr) - { - printf("%s ", msg); - } - printStmtID(stmt); - printf(" ( "); - const DebugInfo& di = stmt->GetDebugInfo(); - // For statements in the root we display just the location without the - // inline context info. - if (di.GetInlineContext() == nullptr || di.GetInlineContext()->IsRoot()) - { - di.GetLocation().Dump(); - } - else - { - stmt->GetDebugInfo().Dump(false); - } - printf(" ... "); - - IL_OFFSET lastILOffs = stmt->GetLastILOffset(); - if (lastILOffs == BAD_IL_OFFSET) - { - printf("???"); - } - else - { - printf("0x%03X", lastILOffs); - } - - printf(" )"); - - DebugInfo par; - if (stmt->GetDebugInfo().GetParent(&par)) - { - printf(" <- "); - par.Dump(true); - } - printf("\n"); - - gtDispTree(stmt->GetRootNode()); -} - -//------------------------------------------------------------------------ -// gtDispBlockStmts: dumps all statements inside `block`. -// -// Arguments: -// block - the block to display statements for. -// -void Compiler::gtDispBlockStmts(BasicBlock* block) -{ - for (Statement* const stmt : block->Statements()) - { - gtDispStmt(stmt); - printf("\n"); - } -} - -//------------------------------------------------------------------------ -// Compiler::gtDispRange: dumps a range of LIR. -// -// Arguments: -// range - the range of LIR to display. -// -void Compiler::gtDispRange(LIR::ReadOnlyRange const& range) -{ - for (GenTree* node : range) - { - gtDispLIRNode(node); - } -} - -//------------------------------------------------------------------------ -// Compiler::gtDispTreeRange: dumps the LIR range that contains all of the -// nodes in the dataflow tree rooted at a given -// node. -// -// Arguments: -// containingRange - the LIR range that contains the root node. -// tree - the root of the dataflow tree. -// -void Compiler::gtDispTreeRange(LIR::Range& containingRange, GenTree* tree) -{ - bool closed; - unsigned sideEffects; - gtDispRange(containingRange.GetTreeRangeWithFlags(tree, &closed, &sideEffects)); -} - -//------------------------------------------------------------------------ -// Compiler::gtDispLIRNode: dumps a single LIR node. -// -// Arguments: -// node - the LIR node to dump. -// prefixMsg - an optional prefix for each line of output. -// -void Compiler::gtDispLIRNode(GenTree* node, const char* prefixMsg /* = nullptr */) -{ - auto displayOperand = [](GenTree* operand, const char* message, IndentInfo operandArc, IndentStack& indentStack, - size_t prefixIndent) { - assert(operand != nullptr); - assert(message != nullptr); - - if (prefixIndent != 0) - { - printf("%*s", (int)prefixIndent, ""); - } - - // 60 spaces for alignment - printf("%-60s", ""); - - indentStack.Push(operandArc); - indentStack.print(); - indentStack.Pop(); - operandArc = IIArc; - - printf(" t%-5d %-6s %s\n", operand->gtTreeID, varTypeName(operand->TypeGet()), message); - }; - - IndentStack indentStack(this); - - size_t prefixIndent = 0; - if (prefixMsg != nullptr) - { - prefixIndent = strlen(prefixMsg); - } - - const int bufLength = 256; - char buf[bufLength]; - - const bool nodeIsCall = node->IsCall(); - - // Visit operands - IndentInfo operandArc = IIArcTop; - for (GenTree* operand : node->Operands()) - { - if (!operand->IsValue()) - { - // Either of these situations may happen with calls. - continue; - } - - if (nodeIsCall) - { - GenTreeCall* call = node->AsCall(); - if (operand == call->gtCallAddr) - { - displayOperand(operand, "calli tgt", operandArc, indentStack, prefixIndent); - } - else if (operand == call->gtControlExpr) - { - displayOperand(operand, "control expr", operandArc, indentStack, prefixIndent); - } - else if (operand == call->gtCallCookie) - { - displayOperand(operand, "cookie", operandArc, indentStack, prefixIndent); - } - else - { - CallArg* curArg = call->gtArgs.FindByNode(operand); - assert(curArg); - - if (operand == curArg->GetEarlyNode()) - { - gtGetArgMsg(call, curArg, buf, sizeof(buf)); - } - else - { - gtGetLateArgMsg(call, curArg, buf, sizeof(buf)); - } - - displayOperand(operand, buf, operandArc, indentStack, prefixIndent); - } - } - else - { - displayOperand(operand, "", operandArc, indentStack, prefixIndent); - } - - operandArc = IIArc; - } - - // Visit the operator - - if (prefixMsg != nullptr) - { - printf("%s", prefixMsg); - } - - const bool topOnly = true; - const bool isLIR = true; - gtDispTree(node, &indentStack, nullptr, topOnly, isLIR); -} - -/*****************************************************************************/ -#endif // DEBUG - -/***************************************************************************** - * - * Check if the given node can be folded, - * and call the methods to perform the folding - */ - -GenTree* Compiler::gtFoldExpr(GenTree* tree) -{ - unsigned kind = tree->OperKind(); - - /* We must have a simple operation to fold */ - - // If we're in CSE, it's not safe to perform tree - // folding given that it can will potentially - // change considered CSE candidates. - if (optValnumCSE_phase) - { - return tree; - } - - if (!opts.Tier0OptimizationEnabled()) - { - return tree; - } - - if (!(kind & GTK_SMPOP)) - { - if (tree->OperIsConditional()) - { - return gtFoldExprConditional(tree); - } - -#if defined(FEATURE_HW_INTRINSICS) - if (tree->OperIsHWIntrinsic()) - { - return gtFoldExprHWIntrinsic(tree->AsHWIntrinsic()); - } -#endif // FEATURE_HW_INTRINSICS - - return tree; - } - - GenTree* op1 = tree->AsOp()->gtOp1; - - /* Filter out non-foldable trees that can have constant children */ - - assert(kind & (GTK_UNOP | GTK_BINOP)); - switch (tree->gtOper) - { - case GT_RETFILT: - case GT_RETURN: - case GT_IND: - return tree; - default: - break; - } - - /* try to fold the current node */ - - if ((kind & GTK_UNOP) && op1) - { - if (op1->OperIsConst()) - { - return gtFoldExprConst(tree); - } - } - else if ((kind & GTK_BINOP) && op1 && tree->AsOp()->gtOp2) - { - GenTree* op2 = tree->AsOp()->gtOp2; - - // The atomic operations are exempted here because they are never computable statically; - // one of their arguments is an address. - if (op1->OperIsConst() && op2->OperIsConst() && !tree->OperIsAtomicOp()) - { - /* both nodes are constants - fold the expression */ - return gtFoldExprConst(tree); - } - else if (op1->OperIsConst() || op2->OperIsConst()) - { - // At least one is a constant - see if we have a - // special operator that can use only one constant - // to fold - e.g. booleans - - if (opts.OptimizationDisabled()) - { - // Too heavy for tier0 - return tree; - } - - return gtFoldExprSpecial(tree); - } - else if (tree->OperIsCompare()) - { - /* comparisons of two local variables can sometimes be folded */ - - return gtFoldExprCompare(tree); - } - } - - /* Return the original node (folded/bashed or not) */ - - return tree; -} - -//------------------------------------------------------------------------ -// gtFoldExprCall: see if a call is foldable -// -// Arguments: -// call - call to examine -// -// Returns: -// The original call if no folding happened. -// An alternative tree if folding happens. -// -// Notes: -// Checks for calls to Type.op_Equality, Type.op_Inequality, and -// Enum.HasFlag, and if the call is to one of these, -// attempts to optimize. - -GenTree* Compiler::gtFoldExprCall(GenTreeCall* call) -{ - // This may discard the call and will thus discard arg setup nodes that may - // have arbitrary side effects, so we only support this being called before - // args have been morphed. - assert(!call->gtArgs.AreArgsComplete()); - - // Can only fold calls to special intrinsics. - if ((call->gtCallMoreFlags & GTF_CALL_M_SPECIAL_INTRINSIC) == 0) - { - return call; - } - - // Defer folding if not optimizing. - if (opts.OptimizationDisabled()) - { - return call; - } - - // Check for a new-style jit intrinsic. - const NamedIntrinsic ni = lookupNamedIntrinsic(call->gtCallMethHnd); - - switch (ni) - { - case NI_System_Enum_HasFlag: - { - GenTree* thisOp = call->gtArgs.GetArgByIndex(0)->GetNode(); - GenTree* flagOp = call->gtArgs.GetArgByIndex(1)->GetNode(); - GenTree* result = gtOptimizeEnumHasFlag(thisOp, flagOp); - - if (result != nullptr) - { - return result; - } - break; - } - - case NI_System_Type_op_Equality: - case NI_System_Type_op_Inequality: - { - noway_assert(call->TypeGet() == TYP_INT); - GenTree* op1 = call->gtArgs.GetArgByIndex(0)->GetNode(); - GenTree* op2 = call->gtArgs.GetArgByIndex(1)->GetNode(); - - // If either operand is known to be a RuntimeType, this can be folded - GenTree* result = gtFoldTypeEqualityCall(ni == NI_System_Type_op_Equality, op1, op2); - if (result != nullptr) - { - return result; - } - break; - } - - default: - break; - } - - return call; -} - -//------------------------------------------------------------------------ -// gtFoldTypeEqualityCall: see if a (potential) type equality call is foldable -// -// Arguments: -// isEq -- is it == or != operator -// op1 -- first argument to call -// op2 -- second argument to call -// -// Returns: -// nulltpr if no folding happened. -// An alternative tree if folding happens. -// -// Notes: -// If either operand is known to be a RuntimeType, then the type -// equality methods will simply check object identity and so we can -// fold the call into a simple compare of the call's operands. - -GenTree* Compiler::gtFoldTypeEqualityCall(bool isEq, GenTree* op1, GenTree* op2) -{ - if ((gtGetTypeProducerKind(op1) == TPK_Unknown) && (gtGetTypeProducerKind(op2) == TPK_Unknown)) - { - return nullptr; - } - - const genTreeOps simpleOp = isEq ? GT_EQ : GT_NE; - - JITDUMP("\nFolding call to Type:op_%s to a simple compare via %s\n", isEq ? "Equality" : "Inequality", - GenTree::OpName(simpleOp)); - - GenTree* compare = gtNewOperNode(simpleOp, TYP_INT, op1, op2); - - return compare; -} - -/***************************************************************************** - * - * Some comparisons can be folded: - * - * locA == locA - * classVarA == classVarA - * locA + locB == locB + locA - * - */ - -GenTree* Compiler::gtFoldExprCompare(GenTree* tree) -{ - GenTree* op1 = tree->AsOp()->gtOp1; - GenTree* op2 = tree->AsOp()->gtOp2; - - assert(tree->OperIsCompare()); - - /* Filter out cases that cannot be folded here */ - - /* Do not fold floats or doubles (e.g. NaN != NaN) */ - - if (varTypeIsFloating(op1->TypeGet())) - { - return tree; - } - - // Currently we can only fold when the two subtrees exactly match - // and everything is side effect free. - // - if (((tree->gtFlags & GTF_SIDE_EFFECT) != 0) || !GenTree::Compare(op1, op2, true)) - { - // No folding. - // - return tree; - } - - // GTF_ORDER_SIDEEFF here may indicate volatile subtrees. - // Or it may indicate a non-null assertion prop into an indir subtree. - // - // Check the operands. - // - if ((tree->gtFlags & GTF_ORDER_SIDEEFF) != 0) - { - // If op1 is "volatle" and op2 is not, we can still fold. - // - const bool op1MayBeVolatile = (op1->gtFlags & GTF_ORDER_SIDEEFF) != 0; - const bool op2MayBeVolatile = (op2->gtFlags & GTF_ORDER_SIDEEFF) != 0; - - if (!op1MayBeVolatile || op2MayBeVolatile) - { - // No folding. - // - return tree; - } - } - - GenTree* cons; - - switch (tree->gtOper) - { - case GT_EQ: - case GT_LE: - case GT_GE: - cons = gtNewIconNode(true); /* Folds to GT_CNS_INT(true) */ - break; - - case GT_NE: - case GT_LT: - case GT_GT: - cons = gtNewIconNode(false); /* Folds to GT_CNS_INT(false) */ - break; - - default: - assert(!"Unexpected relOp"); - return tree; - } - - /* The node has been folded into 'cons' */ - - JITDUMP("\nFolding comparison with identical operands:\n"); - DISPTREE(tree); - - if (fgGlobalMorph) - { - fgMorphTreeDone(cons); - } - else - { - cons->gtNext = tree->gtNext; - cons->gtPrev = tree->gtPrev; - } - - JITDUMP("Bashed to %s:\n", cons->AsIntConCommon()->IconValue() ? "true" : "false"); - DISPTREE(cons); - - return cons; -} - -//------------------------------------------------------------------------ -// gtFoldExprConditional: see if a conditional is foldable -// -// Arguments: -// tree - condition to examine -// -// Returns: -// The original call if no folding happened. -// An alternative tree if folding happens. -// -// Notes: -// Supporting foldings are: -// SELECT TRUE X Y -> X -// SELECT FALSE X Y -> Y -// SELECT COND X X -> X -// -GenTree* Compiler::gtFoldExprConditional(GenTree* tree) -{ - GenTree* cond = tree->AsConditional()->gtCond; - GenTree* op1 = tree->AsConditional()->gtOp1; - GenTree* op2 = tree->AsConditional()->gtOp2; - - assert(tree->OperIsConditional()); - - // Check for a constant conditional - if (cond->OperIsConst()) - { - // Constant conditions must be folded away. - - JITDUMP("\nFolding conditional op with constant condition:\n"); - DISPTREE(tree); - - assert(cond->TypeIs(TYP_INT)); - - GenTree* replacement = nullptr; - if (cond->IsIntegralConst(0)) - { - JITDUMP("Bashed to false path:\n"); - replacement = op2; - } - else - { - // Condition should never be a constant other than 0 or 1 - assert(cond->IsIntegralConst(1)); - JITDUMP("Bashed to true path:\n"); - replacement = op1; - } - - if (fgGlobalMorph) - { - fgMorphTreeDone(replacement); - } - else - { - replacement->gtNext = tree->gtNext; - replacement->gtPrev = tree->gtPrev; - } - DISPTREE(replacement); - JITDUMP("\n"); - - // If we bashed to a compare, try to fold that. - if (replacement->OperIsCompare()) - { - return gtFoldExprCompare(replacement); - } - - return replacement; - } - - assert(cond->OperIsCompare()); - - if (((tree->gtFlags & GTF_SIDE_EFFECT) != 0) || !GenTree::Compare(op1, op2, true)) - { - // No folding. - return tree; - } - - // GTF_ORDER_SIDEEFF here may indicate volatile subtrees. - // Or it may indicate a non-null assertion prop into an indir subtree. - if ((tree->gtFlags & GTF_ORDER_SIDEEFF) != 0) - { - // If op1 is "volatile" and op2 is not, we can still fold. - const bool op1MayBeVolatile = (op1->gtFlags & GTF_ORDER_SIDEEFF) != 0; - const bool op2MayBeVolatile = (op2->gtFlags & GTF_ORDER_SIDEEFF) != 0; - - if (!op1MayBeVolatile || op2MayBeVolatile) - { - // No folding. - return tree; - } - } - - JITDUMP("Bashed to first of two identical paths:\n"); - GenTree* replacement = op1; - - if (fgGlobalMorph) - { - fgMorphTreeDone(replacement); - } - else - { - replacement->gtNext = tree->gtNext; - replacement->gtPrev = tree->gtPrev; - } - DISPTREE(replacement); - JITDUMP("\n"); - - return replacement; -} - -//------------------------------------------------------------------------ -// gtFoldTypeCompare: see if a type comparison can be further simplified -// -// Arguments: -// tree -- tree possibly comparing types -// -// Returns: -// An alternative tree if folding happens. -// Original tree otherwise. -// -// Notes: -// Checks for -// typeof(...) == obj.GetType() -// typeof(...) == typeof(...) -// typeof(...) == null -// obj1.GetType() == obj2.GetType() -// -// And potentially optimizes away the need to obtain actual -// RuntimeType objects to do the comparison. - -GenTree* Compiler::gtFoldTypeCompare(GenTree* tree) -{ - // Only handle EQ and NE - // (maybe relop vs null someday) - const genTreeOps oper = tree->OperGet(); - if ((oper != GT_EQ) && (oper != GT_NE)) - { - return tree; - } - - // Screen for the right kinds of operands - GenTree* const op1 = tree->AsOp()->gtOp1; - GenTree* const op2 = tree->AsOp()->gtOp2; - const TypeProducerKind op1Kind = gtGetTypeProducerKind(op1); - const TypeProducerKind op2Kind = gtGetTypeProducerKind(op2); - - // Fold "typeof(handle) cmp null" - if (((op2Kind == TPK_Null) && (op1Kind == TPK_Handle)) || ((op1Kind == TPK_Null) && (op2Kind == TPK_Handle))) - { - GenTree* call = op1Kind == TPK_Handle ? op1 : op2; - GenTree* handle = call->AsCall()->gtArgs.GetArgByIndex(0)->GetNode(); - if (gtGetHelperArgClassHandle(handle) != NO_CLASS_HANDLE) - { - return oper == GT_EQ ? gtNewFalse() : gtNewTrue(); - } - } - - // If both types are created via handles, we can simply compare - // handles instead of the types that they'd create. - if ((op1Kind == TPK_Handle) && (op2Kind == TPK_Handle)) - { - JITDUMP("Optimizing compare of types-from-handles to instead compare handles\n"); - assert((tree->AsOp()->gtGetOp1()->AsCall()->gtArgs.CountArgs() == 1) && - (tree->AsOp()->gtGetOp2()->AsCall()->gtArgs.CountArgs() == 1)); - GenTree* op1ClassFromHandle = tree->AsOp()->gtGetOp1()->AsCall()->gtArgs.GetArgByIndex(0)->GetNode(); - GenTree* op2ClassFromHandle = tree->AsOp()->gtGetOp2()->AsCall()->gtArgs.GetArgByIndex(0)->GetNode(); - CORINFO_CLASS_HANDLE cls1Hnd = NO_CLASS_HANDLE; - CORINFO_CLASS_HANDLE cls2Hnd = NO_CLASS_HANDLE; - - // Try and find class handles from op1 and op2 - cls1Hnd = gtGetHelperArgClassHandle(op1ClassFromHandle); - cls2Hnd = gtGetHelperArgClassHandle(op2ClassFromHandle); - - // If we have both class handles, try and resolve the type equality test completely. - bool resolveFailed = false; - - if ((cls1Hnd != NO_CLASS_HANDLE) && (cls2Hnd != NO_CLASS_HANDLE)) - { - JITDUMP("Asking runtime to compare %p (%s) and %p (%s) for equality\n", dspPtr(cls1Hnd), - eeGetClassName(cls1Hnd), dspPtr(cls2Hnd), eeGetClassName(cls2Hnd)); - TypeCompareState s = info.compCompHnd->compareTypesForEquality(cls1Hnd, cls2Hnd); - - if (s != TypeCompareState::May) - { - // Type comparison result is known. - const bool typesAreEqual = (s == TypeCompareState::Must); - const bool operatorIsEQ = (oper == GT_EQ); - const int compareResult = operatorIsEQ ^ typesAreEqual ? 0 : 1; - JITDUMP("Runtime reports comparison is known at jit time: %u\n", compareResult); - GenTree* result = gtNewIconNode(compareResult); - return result; - } - else - { - resolveFailed = true; - } - } - - if (resolveFailed) - { - JITDUMP("Runtime reports comparison is NOT known at jit time\n"); - } - else - { - JITDUMP("Could not find handle for %s%s\n", (cls1Hnd == NO_CLASS_HANDLE) ? " cls1" : "", - (cls2Hnd == NO_CLASS_HANDLE) ? " cls2" : ""); - } - - GenTree* compare = gtNewOperNode(oper, TYP_INT, op1ClassFromHandle, op2ClassFromHandle); - - // Drop any now-irrelevant flags - compare->gtFlags |= tree->gtFlags & (GTF_RELOP_JMP_USED | GTF_DONT_CSE); - - return compare; - } - - if ((op1Kind == TPK_GetType) && (op2Kind == TPK_GetType)) - { - GenTree* arg1; - - if (op1->OperGet() == GT_INTRINSIC) - { - arg1 = op1->AsUnOp()->gtOp1; - } - else - { - arg1 = op1->AsCall()->gtArgs.GetThisArg()->GetNode(); - } - - arg1 = gtNewMethodTableLookup(arg1); - - GenTree* arg2; - - if (op2->OperGet() == GT_INTRINSIC) - { - arg2 = op2->AsUnOp()->gtOp1; - } - else - { - arg2 = op2->AsCall()->gtArgs.GetThisArg()->GetNode(); - } - - arg2 = gtNewMethodTableLookup(arg2); - - GenTree* compare = gtNewOperNode(oper, TYP_INT, arg1, arg2); - - // Drop any now-irrelevant flags - compare->gtFlags |= tree->gtFlags & (GTF_RELOP_JMP_USED | GTF_DONT_CSE); - - return compare; - } - - // If one operand creates a type from a handle and the other operand is fetching the type from an object, - // we can sometimes optimize the type compare into a simpler - // method table comparison. - // - // TODO: if other operand is null... - if (!(((op1Kind == TPK_GetType) && (op2Kind == TPK_Handle)) || - ((op1Kind == TPK_Handle) && (op2Kind == TPK_GetType)))) - { - return tree; - } - - GenTree* const opHandle = (op1Kind == TPK_Handle) ? op1 : op2; - GenTree* const opOther = (op1Kind == TPK_Handle) ? op2 : op1; - - // Tunnel through the handle operand to get at the class handle involved. - GenTree* const opHandleArgument = opHandle->AsCall()->gtArgs.GetArgByIndex(0)->GetNode(); - CORINFO_CLASS_HANDLE clsHnd = gtGetHelperArgClassHandle(opHandleArgument); - - // If we couldn't find the class handle, give up. - if (clsHnd == NO_CLASS_HANDLE) - { - return tree; - } - - // Check if an object of this type can even exist - if (info.compCompHnd->getExactClasses(clsHnd, 0, nullptr) == 0) - { - JITDUMP("Runtime reported %p (%s) is never allocated\n", dspPtr(clsHnd), eeGetClassName(clsHnd)); - - const bool operatorIsEQ = (oper == GT_EQ); - const int compareResult = operatorIsEQ ? 0 : 1; - JITDUMP("Runtime reports comparison is known at jit time: %u\n", compareResult); - GenTree* result = gtNewIconNode(compareResult); - return result; - } - - // We're good to go. - JITDUMP("Optimizing compare of obj.GetType()" - " and type-from-handle to compare method table pointer\n"); - - // opHandleArgument is the method table we're looking for. - GenTree* const knownMT = opHandleArgument; - - // Fetch object method table from the object itself. - GenTree* objOp = nullptr; - - // Note we may see intrinsified or regular calls to GetType - if (opOther->OperGet() == GT_INTRINSIC) - { - objOp = opOther->AsUnOp()->gtOp1; - } - else - { - objOp = opOther->AsCall()->gtArgs.GetThisArg()->GetNode(); - } - - bool isExact = false; - bool isNonNull = false; - CORINFO_CLASS_HANDLE objCls = gtGetClassHandle(objOp, &isExact, &isNonNull); - - // if both classes are "final" (e.g. System.String[]) we can replace the comparison - // with `true/false` + null check. - if ((objCls != NO_CLASS_HANDLE) && (isExact || info.compCompHnd->isExactType(objCls))) - { - TypeCompareState tcs = info.compCompHnd->compareTypesForEquality(objCls, clsHnd); - if (tcs != TypeCompareState::May) - { - const bool operatorIsEQ = oper == GT_EQ; - const bool typesAreEqual = tcs == TypeCompareState::Must; - GenTree* compareResult = gtNewIconNode((operatorIsEQ ^ typesAreEqual) ? 0 : 1); - - if (!isNonNull) - { - // we still have to emit a null-check - // obj.GetType == typeof() -> (nullcheck) true/false - GenTree* nullcheck = gtNewNullCheck(objOp, compCurBB); - return gtNewOperNode(GT_COMMA, tree->TypeGet(), nullcheck, compareResult); - } - else if (objOp->gtFlags & GTF_ALL_EFFECT) - { - return gtNewOperNode(GT_COMMA, tree->TypeGet(), objOp, compareResult); - } - else - { - return compareResult; - } - } - } - - // Fetch the method table from the object - GenTree* const objMT = gtNewMethodTableLookup(objOp); - - // Compare the two method tables - GenTree* const compare = gtNewOperNode(oper, TYP_INT, objMT, knownMT); - - // Drop any now irrelevant flags - compare->gtFlags |= tree->gtFlags & (GTF_RELOP_JMP_USED | GTF_DONT_CSE); - - // And we're done - return compare; -} - -//------------------------------------------------------------------------ -// gtGetHelperArgClassHandle: find the compile time class handle from -// a helper call argument tree -// -// Arguments: -// tree - tree that passes the handle to the helper -// -// Returns: -// The compile time class handle if known. -// -CORINFO_CLASS_HANDLE Compiler::gtGetHelperArgClassHandle(GenTree* tree) -{ - CORINFO_CLASS_HANDLE result = NO_CLASS_HANDLE; - - // The handle could be a literal constant - if ((tree->OperGet() == GT_CNS_INT) && (tree->TypeGet() == TYP_I_IMPL)) - { - assert(tree->IsIconHandle(GTF_ICON_CLASS_HDL)); - result = (CORINFO_CLASS_HANDLE)tree->AsIntCon()->gtCompileTimeHandle; - } - // Or the result of a runtime lookup - else if (tree->OperGet() == GT_RUNTIMELOOKUP) - { - result = tree->AsRuntimeLookup()->GetClassHandle(); - } - // Or something reached indirectly - else if (tree->gtOper == GT_IND) - { - // The handle indirs we are looking for will be marked as non-faulting. - // Certain others (eg from refanytype) may not be. - if (tree->gtFlags & GTF_IND_NONFAULTING) - { - GenTree* handleTreeInternal = tree->AsOp()->gtOp1; - - if ((handleTreeInternal->OperGet() == GT_CNS_INT) && (handleTreeInternal->TypeGet() == TYP_I_IMPL)) - { - // These handle constants should be class handles. - assert(handleTreeInternal->IsIconHandle(GTF_ICON_CLASS_HDL)); - result = (CORINFO_CLASS_HANDLE)handleTreeInternal->AsIntCon()->gtCompileTimeHandle; - } - } - } - - return result; -} - -//------------------------------------------------------------------------ -// gtGetHelperArgMethodHandle: find the compile time method handle from -// a helper call argument tree -// -// Arguments: -// tree - tree that passes the handle to the helper -// -// Returns: -// The compile time method handle, if known. -// -CORINFO_METHOD_HANDLE Compiler::gtGetHelperArgMethodHandle(GenTree* tree) -{ - CORINFO_METHOD_HANDLE result = NO_METHOD_HANDLE; - - // The handle could be a literal constant - if ((tree->OperGet() == GT_CNS_INT) && (tree->TypeGet() == TYP_I_IMPL)) - { - assert(tree->IsIconHandle(GTF_ICON_METHOD_HDL)); - result = (CORINFO_METHOD_HANDLE)tree->AsIntCon()->gtCompileTimeHandle; - } - // Or the result of a runtime lookup - else if (tree->OperGet() == GT_RUNTIMELOOKUP) - { - result = tree->AsRuntimeLookup()->GetMethodHandle(); - } - // Or something reached indirectly - else if (tree->gtOper == GT_IND) - { - // The handle indirs we are looking for will be marked as non-faulting. - // Certain others (eg from refanytype) may not be. - if (tree->gtFlags & GTF_IND_NONFAULTING) - { - GenTree* handleTreeInternal = tree->AsOp()->gtOp1; - - if ((handleTreeInternal->OperGet() == GT_CNS_INT) && (handleTreeInternal->TypeGet() == TYP_I_IMPL)) - { - // These handle constants should be method handles. - assert(handleTreeInternal->IsIconHandle(GTF_ICON_METHOD_HDL)); - result = (CORINFO_METHOD_HANDLE)handleTreeInternal->AsIntCon()->gtCompileTimeHandle; - } - } - } - - return result; -} - -//------------------------------------------------------------------------ -// gtFoldExprSpecial -- optimize binary ops with one constant operand -// -// Arguments: -// tree - tree to optimize -// -// Return value: -// Tree (possibly modified at root or below), or a new tree -// Any new tree is fully morphed, if necessary. -// -GenTree* Compiler::gtFoldExprSpecial(GenTree* tree) -{ - var_types type = tree->TypeGet(); - GenTree* op1 = tree->AsOp()->gtOp1; - GenTree* op2 = tree->AsOp()->gtOp2; - genTreeOps oper = tree->OperGet(); - - GenTree* op; - GenTree* cons; - ssize_t val; - - assert(tree->OperKind() & GTK_BINOP); - - /* Filter out operators that cannot be folded here */ - if (oper == GT_CAST) - { - return tree; - } - - /* We only consider TYP_INT for folding - * Do not fold pointer arithmetic (e.g. addressing modes!) */ - - if (oper != GT_QMARK && !varTypeIsIntOrI(type)) - { - if (varTypeIsFloating(type)) - { - return gtFoldExprSpecialFloating(tree); - } - return tree; - } - - /* Find out which is the constant node */ - - if (op1->IsCnsIntOrI()) - { - op = op2; - cons = op1; - } - else if (op2->IsCnsIntOrI()) - { - op = op1; - cons = op2; - } - else - { - return tree; - } - - /* Get the constant value */ - - val = cons->AsIntConCommon()->IconValue(); - - // Helper function that creates a new IntCon node and morphs it, if required - auto NewMorphedIntConNode = [&](int value) -> GenTreeIntCon* { - GenTreeIntCon* icon = gtNewIconNode(value); - icon->SetMorphed(this); - return icon; - }; - - auto NewZeroExtendNode = [&](var_types type, GenTree* op1, var_types castToType) -> GenTree* { - assert(varTypeIsIntegral(type)); - assert(!varTypeIsSmall(type)); - assert(!varTypeIsUnsigned(type)); - assert(varTypeIsUnsigned(castToType)); - - GenTreeCast* cast = gtNewCastNode(TYP_INT, op1, false, castToType); - cast->SetMorphed(this); - fgMorphTreeDone(cast); - - if (type == TYP_LONG) - { - cast = gtNewCastNode(TYP_LONG, cast, true, TYP_LONG); - cast->SetMorphed(this); - fgMorphTreeDone(cast); - } - - return cast; - }; - - // Here `op` is the non-constant operand, `cons` is the constant operand - // and `val` is the constant value. - - if (((op->gtFlags & GTF_SIDE_EFFECT) != 0) && tree->OperIsCompare() && ((tree->gtFlags & GTF_RELOP_JMP_USED) != 0)) - { - // TODO-CQ: Some phases currently have an invariant that JTRUE(x) - // must have x be a relational operator. As such, we cannot currently - // fold such cases and need to preserve the tree as is. - return tree; - } - - switch (oper) - { - case GT_LE: - { - if (tree->IsUnsigned() && (val == 0) && (op1 == cons)) - { - // unsigned (0 <= x) is always true - op = gtWrapWithSideEffects(NewMorphedIntConNode(1), op, GTF_ALL_EFFECT); - goto DONE_FOLD; - } - break; - } - - case GT_GE: - { - if (tree->IsUnsigned() && (val == 0) && (op2 == cons)) - { - // unsigned (x >= 0) is always true - op = gtWrapWithSideEffects(NewMorphedIntConNode(1), op, GTF_ALL_EFFECT); - goto DONE_FOLD; - } - break; - } - - case GT_LT: - { - if (tree->IsUnsigned() && (val == 0) && (op2 == cons)) - { - // unsigned (x < 0) is always false - op = gtWrapWithSideEffects(NewMorphedIntConNode(0), op, GTF_ALL_EFFECT); - goto DONE_FOLD; - } - break; - } - - case GT_GT: - { - if (tree->IsUnsigned() && (val == 0) && (op1 == cons)) - { - // unsigned (0 > x) is always false - op = gtWrapWithSideEffects(NewMorphedIntConNode(0), op, GTF_ALL_EFFECT); - goto DONE_FOLD; - } - } - FALLTHROUGH; - - case GT_EQ: - case GT_NE: - { - // For "obj ==/!= null" we can fold the comparison to true/false if VM tells us that the object cannot - // be non-null. Example: "obj == null" -> true if obj's type is FooCls and FooCls is an abstract class - // without subclasses. - // - // IsTargetAbi check is not necessary here, but it improves TP for runtimes where this optimization is - // not possible anyway. - if ((val == 0) && op->TypeIs(TYP_REF) && IsTargetAbi(CORINFO_NATIVEAOT_ABI)) - { - bool isExact, isNonNull; - CORINFO_CLASS_HANDLE opCls = gtGetClassHandle(op, &isExact, &isNonNull); - if ((opCls != NO_CLASS_HANDLE) && (info.compCompHnd->getExactClasses(opCls, 0, nullptr) == 0)) - { - op = gtWrapWithSideEffects(NewMorphedIntConNode(tree->OperIs(GT_EQ) ? 1 : 0), op, GTF_ALL_EFFECT); - goto DONE_FOLD; - } - } - - // Optimize boxed value classes; these are always false. This IL is - // generated when a generic value is tested against null: - // ... foo(T x) { ... if ((object)x == null) ... - if ((val == 0) && op->IsBoxedValue()) - { - JITDUMP("\nAttempting to optimize BOX(valueType) %s null [%06u]\n", GenTree::OpName(oper), - dspTreeID(tree)); - - // We don't expect GT_GT with signed compares, and we - // can't predict the result if we do see it, since the - // boxed object addr could have its high bit set. - if ((oper == GT_GT) && !tree->IsUnsigned()) - { - JITDUMP(" bailing; unexpected signed compare via GT_GT\n"); - } - else - { - // The tree under the box must be side effect free - // since we will drop it if we optimize. - assert(!gtTreeHasSideEffects(op->AsBox()->BoxOp(), GTF_SIDE_EFFECT)); - - // See if we can optimize away the box and related statements. - GenTree* boxSourceTree = gtTryRemoveBoxUpstreamEffects(op); - bool didOptimize = (boxSourceTree != nullptr); - - // If optimization succeeded, remove the box. - if (didOptimize) - { - // Set up the result of the compare. - int compareResult = 0; - if (oper == GT_GT) - { - // GT_GT(null, box) == false - // GT_GT(box, null) == true - compareResult = (op1 == op); - } - else if (oper == GT_EQ) - { - // GT_EQ(box, null) == false - // GT_EQ(null, box) == false - compareResult = 0; - } - else - { - assert(oper == GT_NE); - // GT_NE(box, null) == true - // GT_NE(null, box) == true - compareResult = 1; - } - - JITDUMP("\nSuccess: replacing BOX(valueType) %s null with %d\n", GenTree::OpName(oper), - compareResult); - - return NewMorphedIntConNode(compareResult); - } - } - } - else - { - return gtFoldBoxNullable(tree); - } - break; - } - - case GT_ADD: - { - if (val == 0) - { - goto DONE_FOLD; - } - break; - } - - case GT_MUL: - { - if (val == 1) - { - goto DONE_FOLD; - } - else if (val == 0) - { - // Multiply by zero - return the 'zero' node - op = gtWrapWithSideEffects(cons, op, GTF_ALL_EFFECT); - goto DONE_FOLD; - } - break; - } - - case GT_DIV: - case GT_UDIV: - { - if ((op2 == cons) && (val == 1) && !op1->OperIsConst()) - { - goto DONE_FOLD; - } - break; - } - - case GT_SUB: - { - if ((op2 == cons) && (val == 0) && !op1->OperIsConst()) - { - goto DONE_FOLD; - } - break; - } - - case GT_AND: - { - if (val == 0) - { - // AND with zero - return the 'zero' node - op = gtWrapWithSideEffects(cons, op, GTF_ALL_EFFECT); - goto DONE_FOLD; - } - else if (val == 0xFF) - { - op = NewZeroExtendNode(tree->TypeGet(), op, TYP_UBYTE); - goto DONE_FOLD; - } - else if (val == 0xFFFF) - { - op = NewZeroExtendNode(tree->TypeGet(), op, TYP_USHORT); - goto DONE_FOLD; - } - else if ((val == 0xFFFFFFFF) && varTypeIsLong(tree)) - { - op = NewZeroExtendNode(tree->TypeGet(), op, TYP_UINT); - goto DONE_FOLD; - } - break; - } - - case GT_OR: - { - if (val == 0) - { - goto DONE_FOLD; - } - break; - } - - case GT_LSH: - case GT_RSH: - case GT_RSZ: - case GT_ROL: - case GT_ROR: - { - if (val == 0) - { - if (op2 == cons) - { - goto DONE_FOLD; - } - else - { - op = gtWrapWithSideEffects(cons, op, GTF_ALL_EFFECT); - goto DONE_FOLD; - } - } - break; - } - - case GT_QMARK: - { - assert(op1 == cons && op2 == op && op2->gtOper == GT_COLON); - assert(op2->AsOp()->gtOp1 && op2->AsOp()->gtOp2); - - assert(val == 0 || val == 1); - - if (val) - { - op = op2->AsColon()->ThenNode(); - } - else - { - op = op2->AsColon()->ElseNode(); - } - - // Clear colon flags only if the qmark itself is not conditionally executed - if ((tree->gtFlags & GTF_COLON_COND) == 0) - { - fgWalkTreePre(&op, gtClearColonCond); - } - goto DONE_FOLD; - } - - default: - break; - } - - /* The node is not foldable */ - - return tree; - -DONE_FOLD: - - JITDUMP("\nFolding binary operator with a constant operand:\n"); - DISPTREE(tree); - JITDUMP("Transformed into:\n"); - DISPTREE(op); - op->SetMorphed(this); - return op; -} - -//------------------------------------------------------------------------ -// gtFoldExprSpecialFloating -- optimize floating-point binary ops with one constant operand -// -// Arguments: -// tree - tree to optimize -// -// Return value: -// Tree (possibly modified at root or below), or a new tree -// Any new tree is fully morphed, if necessary. -// -GenTree* Compiler::gtFoldExprSpecialFloating(GenTree* tree) -{ - assert(varTypeIsFloating(tree->TypeGet())); - assert(tree->OperKind() & GTK_BINOP); - - GenTree* op1 = tree->AsOp()->gtOp1; - GenTree* op2 = tree->AsOp()->gtOp2; - genTreeOps oper = tree->OperGet(); - - GenTree* op; - GenTree* cons; - double val; - - /* Filter out operators that cannot be folded here */ - if (oper == GT_CAST) - { - return tree; - } - - /* Find out which is the constant node */ - if (op1->IsCnsFltOrDbl()) - { - op = op2; - cons = op1; - } - else if (op2->IsCnsFltOrDbl()) - { - op = op1; - cons = op2; - } - else - { - return tree; - } - - /* Get the constant value */ - val = cons->AsDblCon()->DconValue(); - - // Helper function that creates a new IntCon node and morphs it, if required - auto NewMorphedIntConNode = [&](int value) -> GenTreeIntCon* { - GenTreeIntCon* icon = gtNewIconNode(value); - - icon->SetMorphed(this); - - return icon; - }; - - // Here `op` is the non-constant operand, `cons` is the constant operand - // and `val` is the constant value. - - switch (oper) - { - case GT_ADD: - { - // Handle `x + NaN == NaN` and `NaN + x == NaN` - // This is safe for all floats since we do not fault for sNaN - - if (FloatingPointUtils::isNaN(val)) - { - op = gtWrapWithSideEffects(cons, op, GTF_ALL_EFFECT); - goto DONE_FOLD; - } - - // Handle `x + -0 == x` and `-0 + x == x` - - if (FloatingPointUtils::isNegativeZero(val)) - { - goto DONE_FOLD; - } - - // We cannot handle `x + 0 == x` or `0 + x == x` since `-0 + 0 == 0` - break; - } - - case GT_DIV: - { - // Handle `x / NaN == NaN` and `NaN / x == NaN` - // This is safe for all floats since we do not fault for sNaN - - if (FloatingPointUtils::isNaN(val)) - { - op = gtWrapWithSideEffects(cons, op, GTF_ALL_EFFECT); - goto DONE_FOLD; - } - - // Handle `x / 1 == x`. - // This is safe for all floats since we do not fault for sNaN - - if ((op2 == cons) && (val == 1.0)) - { - goto DONE_FOLD; - } - break; - } - - case GT_EQ: - { - assert((tree->gtFlags & GTF_RELOP_NAN_UN) == 0); - - if (FloatingPointUtils::isNaN(val)) - { - // Comparison with NaN is always false - op = gtWrapWithSideEffects(NewMorphedIntConNode(0), op, GTF_ALL_EFFECT); - goto DONE_FOLD; - } - break; - } - - case GT_GE: - case GT_GT: - case GT_LE: - case GT_LT: - { - if (FloatingPointUtils::isNaN(val)) - { - if ((tree->gtFlags & GTF_RELOP_NAN_UN) != 0) - { - // Unordered comparison with NaN is always true - op = gtWrapWithSideEffects(NewMorphedIntConNode(1), op, GTF_ALL_EFFECT); - } - else - { - // Comparison with NaN is always false - op = gtWrapWithSideEffects(NewMorphedIntConNode(0), op, GTF_ALL_EFFECT); - } - goto DONE_FOLD; - } - break; - } - - case GT_MUL: - { - // Handle `x * NaN == NaN` and `NaN * x == NaN` - // This is safe for all floats since we do not fault for sNaN - - if (FloatingPointUtils::isNaN(val)) - { - op = gtWrapWithSideEffects(cons, op, GTF_ALL_EFFECT); - goto DONE_FOLD; - } - - // Handle `x * 1 == x` and `1 * x == x` - // This is safe for all floats since we do not fault for sNaN - - if (val == 1.0) - { - goto DONE_FOLD; - } - - // We cannot handle `x * 0 == 0` or ` 0 * x == 0` since `-0 * 0 == -0` - // We cannot handle `x * -0 == -0` or `-0 * x == -0` since `-0 * -0 == 0` - break; - } - - case GT_NE: - { - assert((tree->gtFlags & GTF_RELOP_NAN_UN) == 0); - - if (FloatingPointUtils::isNaN(val)) - { - // Comparison with NaN is always true - op = gtWrapWithSideEffects(NewMorphedIntConNode(1), op, GTF_ALL_EFFECT); - goto DONE_FOLD; - } - break; - } - - case GT_SUB: - { - // Handle `x - NaN == NaN` and `NaN - x == NaN` - // This is safe for all floats since we do not fault for sNaN - - if (FloatingPointUtils::isNaN(val)) - { - op = gtWrapWithSideEffects(cons, op, GTF_ALL_EFFECT); - goto DONE_FOLD; - } - - // Handle `x - 0 == x` - - if ((cons == op2) && FloatingPointUtils::isPositiveZero(val)) - { - goto DONE_FOLD; - } - - // We cannot handle `x - -0 == x` since `-0 - -0 == 0` - break; - } - - default: - { - break; - } - } - - /* The node is not foldable */ - - return tree; - -DONE_FOLD: - - JITDUMP("\nFolding binary operator with a constant operand:\n"); - DISPTREE(tree); - JITDUMP("Transformed into:\n"); - DISPTREE(op); - op->SetMorphed(this); - - return op; -} - -//------------------------------------------------------------------------ -// gtFoldBoxNullable -- optimize a boxed nullable feeding a compare to zero -// -// Arguments: -// tree - binop tree to potentially optimize, must be -// GT_GT, GT_EQ, or GT_NE -// -// Return value: -// Tree (possibly modified below the root). -// -GenTree* Compiler::gtFoldBoxNullable(GenTree* tree) -{ - assert(tree->OperKind() & GTK_BINOP); - assert(tree->OperIs(GT_GT, GT_EQ, GT_NE)); - - genTreeOps const oper = tree->OperGet(); - - if ((oper == GT_GT) && !tree->IsUnsigned()) - { - return tree; - } - - GenTree* const op1 = tree->AsOp()->gtOp1; - GenTree* const op2 = tree->AsOp()->gtOp2; - GenTree* op; - GenTree* cons; - - if (op1->IsCnsIntOrI()) - { - op = op2; - cons = op1; - } - else if (op2->IsCnsIntOrI()) - { - op = op1; - cons = op2; - } - else - { - return tree; - } - - ssize_t const val = cons->AsIntConCommon()->IconValue(); - - if (val != 0) - { - return tree; - } - - if (!op->IsCall()) - { - return tree; - } - - GenTreeCall* const call = op->AsCall(); - - if (!call->IsHelperCall(this, CORINFO_HELP_BOX_NULLABLE)) - { - return tree; - } - - if (call->gtArgs.AreArgsComplete()) - { - // We cannot handle folding the call away when remorphing. - return tree; - } - - JITDUMP("\nReplacing BOX_NULLABLE(&x) %s null [%06u] with x.hasValue\n", GenTree::OpName(oper), dspTreeID(tree)); - - static_assert_no_msg(OFFSETOF__CORINFO_NullableOfT__hasValue == 0); - GenTree* srcAddr = call->gtArgs.GetArgByIndex(1)->GetNode(); - GenTree* hasValueNode = gtNewIndir(TYP_UBYTE, srcAddr); - - if (op == op1) - { - tree->AsOp()->gtOp1 = hasValueNode; - } - else - { - tree->AsOp()->gtOp2 = hasValueNode; - } - - cons->gtType = TYP_INT; - - return tree; -} - -//------------------------------------------------------------------------ -// gtTryRemoveBoxUpstreamEffects: given an unused value type box, -// try and remove the upstream allocation and unnecessary parts of -// the copy. -// -// Arguments: -// op - the box node to optimize -// options - controls whether and how trees are modified -// (see notes) -// -// Return Value: -// A tree representing the original value to box, if removal -// is successful/possible (but see note). nullptr if removal fails. -// -// Notes: -// Value typed box gets special treatment because it has associated -// side effects that can be removed if the box result is not used. -// -// By default (options == BR_REMOVE_AND_NARROW) this method will -// try and remove unnecessary trees and will try and reduce remaining -// operations to the minimal set, possibly narrowing the width of -// loads from the box source if it is a struct. -// -// To perform a trial removal, pass BR_DONT_REMOVE. This can be -// useful to determine if this optimization should only be -// performed if some other conditions hold true. -// -// To remove but not alter the access to the box source, pass -// BR_REMOVE_BUT_NOT_NARROW. -// -// To remove and return the tree for the type handle used for -// the boxed newobj, pass BR_REMOVE_BUT_NOT_NARROW_WANT_TYPE_HANDLE. -// This can be useful when the only part of the box that is "live" -// is its type. -// -// If removal fails, it is possible that a subsequent pass may be -// able to optimize. Blocking side effects may now be minimized -// (null or bounds checks might have been removed) or might be -// better known (inline return placeholder updated with the actual -// return expression). So the box is perhaps best left as is to -// help trigger this re-examination. - -GenTree* Compiler::gtTryRemoveBoxUpstreamEffects(GenTree* op, BoxRemovalOptions options) -{ - assert(op->IsBoxedValue()); - - // grab related parts for the optimization - GenTreeBox* box = op->AsBox(); - Statement* allocStmt = box->gtDefStmtWhenInlinedBoxValue; - Statement* copyStmt = box->gtCopyStmtWhenInlinedBoxValue; - - JITDUMP("gtTryRemoveBoxUpstreamEffects: %s to %s of BOX (valuetype)" - " [%06u] (assign/newobj " FMT_STMT " copy " FMT_STMT "\n", - (options == BR_DONT_REMOVE) ? "checking if it is possible" : "attempting", - (options == BR_MAKE_LOCAL_COPY) ? "make local unboxed version" : "remove side effects", dspTreeID(op), - allocStmt->GetID(), copyStmt->GetID()); - - // If we don't recognize the form of the store, bail. - GenTree* boxLclDef = allocStmt->GetRootNode(); - if (!boxLclDef->OperIs(GT_STORE_LCL_VAR)) - { - JITDUMP(" bailing; unexpected alloc def op %s\n", GenTree::OpName(boxLclDef->OperGet())); - return nullptr; - } - - // If this box is no longer single-use, bail. - if (box->WasCloned()) - { - JITDUMP(" bailing; unsafe to remove box that has been cloned\n"); - return nullptr; - } - - // If we're eventually going to return the type handle, remember it now. - GenTree* boxTypeHandle = nullptr; - if ((options == BR_REMOVE_AND_NARROW_WANT_TYPE_HANDLE) || (options == BR_DONT_REMOVE_WANT_TYPE_HANDLE)) - { - GenTree* defSrc = boxLclDef->AsLclVar()->Data(); - genTreeOps defSrcOper = defSrc->OperGet(); - - // Allocation may be via AllocObj or via helper call, depending - // on when this is invoked and whether the jit is using AllocObj - // for R2R allocations. - if (defSrcOper == GT_ALLOCOBJ) - { - GenTreeAllocObj* allocObj = defSrc->AsAllocObj(); - boxTypeHandle = allocObj->AsOp()->gtOp1; - } - else if (defSrcOper == GT_CALL) - { - GenTreeCall* newobjCall = defSrc->AsCall(); - - // In R2R expansions the handle may not be an explicit operand to the helper, - // so we can't remove the box. - if (newobjCall->gtArgs.IsEmpty()) - { - assert(newobjCall->IsHelperCall(this, CORINFO_HELP_READYTORUN_NEW)); - JITDUMP(" bailing; newobj via R2R helper\n"); - return nullptr; - } - - boxTypeHandle = newobjCall->gtArgs.GetArgByIndex(0)->GetNode(); - } - else - { - unreached(); - } - - assert(boxTypeHandle != nullptr); - } - - // If we don't recognize the form of the copy, bail. - GenTree* copy = copyStmt->GetRootNode(); - if (!copy->OperIs(GT_STOREIND, GT_STORE_BLK)) - { - // GT_RET_EXPR is a tolerable temporary failure. - // The jit will revisit this optimization after - // inlining is done. - if (copy->gtOper == GT_RET_EXPR) - { - JITDUMP(" bailing; must wait for replacement of copy %s\n", GenTree::OpName(copy->gtOper)); - } - else - { - // Anything else is a missed case we should figure out how to handle. - // One known case is GT_COMMAs enclosing the store we are looking for. - JITDUMP(" bailing; unexpected copy op %s\n", GenTree::OpName(copy->gtOper)); - } - return nullptr; - } - - // Handle case where we are optimizing the box into a local copy - if (options == BR_MAKE_LOCAL_COPY) - { - // Drill into the box to get at the box temp local and the box type - GenTree* boxTemp = box->BoxOp(); - assert(boxTemp->IsLocal()); - const unsigned boxTempLcl = boxTemp->AsLclVar()->GetLclNum(); - assert(lvaTable[boxTempLcl].lvType == TYP_REF); - CORINFO_CLASS_HANDLE boxClass = lvaTable[boxTempLcl].lvClassHnd; - assert(boxClass != nullptr); - - // Verify that the copy has the expected shape - // (store_blk|store_ind (add (boxTempLcl, ptr-size))) - // - // The shape here is constrained to the patterns we produce - // over in impImportAndPushBox for the inlined box case. - // - GenTree* copyDstAddr = copy->AsIndir()->Addr(); - if (copyDstAddr->OperGet() != GT_ADD) - { - JITDUMP("Unexpected copy dest address tree\n"); - return nullptr; - } - - GenTree* copyDstAddrOp1 = copyDstAddr->AsOp()->gtOp1; - if ((copyDstAddrOp1->OperGet() != GT_LCL_VAR) || (copyDstAddrOp1->AsLclVarCommon()->GetLclNum() != boxTempLcl)) - { - JITDUMP("Unexpected copy dest address 1st addend\n"); - return nullptr; - } - - GenTree* copyDstAddrOp2 = copyDstAddr->AsOp()->gtOp2; - if (!copyDstAddrOp2->IsIntegralConst(TARGET_POINTER_SIZE)) - { - JITDUMP("Unexpected copy dest address 2nd addend\n"); - return nullptr; - } - - // Screening checks have all passed. Do the transformation. - // - // Retype the box temp to be a struct - JITDUMP("Retyping box temp V%02u to struct %s\n", boxTempLcl, eeGetClassName(boxClass)); - lvaTable[boxTempLcl].lvType = TYP_UNDEF; - const bool isUnsafeValueClass = false; - lvaSetStruct(boxTempLcl, boxClass, isUnsafeValueClass); - - // Remove the newobj and store to box temp - JITDUMP("Bashing NEWOBJ [%06u] to NOP\n", dspTreeID(boxLclDef)); - boxLclDef->gtBashToNOP(); - - // Update the copy from the value to be boxed to the box temp - copy->AsIndir()->Addr() = gtNewLclVarAddrNode(boxTempLcl, TYP_BYREF); - - // Return the address of the now-struct typed box temp - GenTree* retValue = gtNewLclVarAddrNode(boxTempLcl, TYP_BYREF); - - return retValue; - } - - // If the copy is a struct copy, make sure we know how to isolate any source side effects. - GenTree* copySrc = copy->Data(); - - // If the copy source is from a pending inline, wait for it to resolve. - if (copySrc->gtOper == GT_RET_EXPR) - { - JITDUMP(" bailing; must wait for replacement of copy source %s\n", GenTree::OpName(copySrc->gtOper)); - return nullptr; - } - - bool hasSrcSideEffect = false; - bool isStructCopy = false; - - if (gtTreeHasSideEffects(copySrc, GTF_SIDE_EFFECT)) - { - hasSrcSideEffect = true; - - if (varTypeIsStruct(copySrc)) - { - isStructCopy = true; - - if (!copySrc->OperIs(GT_IND, GT_BLK)) - { - // We don't know how to handle other cases, yet. - JITDUMP(" bailing; unexpected copy source struct op with side effect %s\n", - GenTree::OpName(copySrc->gtOper)); - return nullptr; - } - } - } - - // If this was a trial removal, we're done. - if (options == BR_DONT_REMOVE) - { - return copySrc; - } - - if (options == BR_DONT_REMOVE_WANT_TYPE_HANDLE) - { - return boxTypeHandle; - } - - // Otherwise, proceed with the optimization. - // - // Change the store expression to a NOP. - JITDUMP("\nBashing NEWOBJ [%06u] to NOP\n", dspTreeID(boxLclDef)); - boxLclDef->gtBashToNOP(); - - // Change the copy expression so it preserves key - // source side effects. - JITDUMP("\nBashing COPY [%06u]", dspTreeID(copy)); - - if (!hasSrcSideEffect) - { - // If there were no copy source side effects just bash - // the copy to a NOP. - copy->gtBashToNOP(); - JITDUMP(" to NOP; no source side effects.\n"); - } - else if (!isStructCopy) - { - // For scalar types, go ahead and produce the - // value as the copy is fairly cheap and likely - // the optimizer can trim things down to just the - // minimal side effect parts. - copyStmt->SetRootNode(copySrc); - JITDUMP(" to scalar read via [%06u]\n", dspTreeID(copySrc)); - } - else - { - // For struct types read the first byte of the source struct; there's - // no need to read the entire thing, and no place to put it. - assert(copySrc->OperIs(GT_BLK, GT_IND)); - copyStmt->SetRootNode(copySrc); - - if (options == BR_REMOVE_AND_NARROW || options == BR_REMOVE_AND_NARROW_WANT_TYPE_HANDLE) - { - JITDUMP(" to read first byte of struct via modified [%06u]\n", dspTreeID(copySrc)); - copySrc->ChangeOper(GT_IND); - copySrc->ChangeType(TYP_BYTE); - } - else - { - JITDUMP(" to read entire struct via modified [%06u]\n", dspTreeID(copySrc)); - } - } - - if (fgNodeThreading == NodeThreading::AllTrees) - { - fgSetStmtSeq(allocStmt); - fgSetStmtSeq(copyStmt); - } - - // Box effects were successfully optimized. - - if (options == BR_REMOVE_AND_NARROW_WANT_TYPE_HANDLE) - { - return boxTypeHandle; - } - else - { - return copySrc; - } -} - -//------------------------------------------------------------------------ -// gtOptimizeEnumHasFlag: given the operands for a call to Enum.HasFlag, -// try and optimize the call to a simple and/compare tree. -// -// Arguments: -// thisOp - first argument to the call -// flagOp - second argument to the call -// -// Return Value: -// A new cmp/amd tree if successful. nullptr on failure. -// -// Notes: -// If successful, may allocate new temps and modify connected -// statements. - -GenTree* Compiler::gtOptimizeEnumHasFlag(GenTree* thisOp, GenTree* flagOp) -{ - JITDUMP("Considering optimizing call to Enum.HasFlag....\n"); - - // Operands must be boxes - if (!thisOp->IsBoxedValue() || !flagOp->IsBoxedValue()) - { - JITDUMP("bailing, need both inputs to be BOXes\n"); - return nullptr; - } - - // Operands must have same type - bool isExactThis = false; - bool isNonNullThis = false; - CORINFO_CLASS_HANDLE thisHnd = gtGetClassHandle(thisOp, &isExactThis, &isNonNullThis); - - if (thisHnd == nullptr) - { - JITDUMP("bailing, can't find type for 'this' operand\n"); - return nullptr; - } - - // A boxed thisOp should have exact type and non-null instance - assert(isExactThis); - assert(isNonNullThis); - - bool isExactFlag = false; - bool isNonNullFlag = false; - CORINFO_CLASS_HANDLE flagHnd = gtGetClassHandle(flagOp, &isExactFlag, &isNonNullFlag); - - if (flagHnd == nullptr) - { - JITDUMP("bailing, can't find type for 'flag' operand\n"); - return nullptr; - } - - // A boxed flagOp should have exact type and non-null instance - assert(isExactFlag); - assert(isNonNullFlag); - - if (flagHnd != thisHnd) - { - JITDUMP("bailing, operand types differ\n"); - return nullptr; - } - - // If we have a shared type instance we can't safely check type - // equality, so bail. - if (eeIsSharedInst(thisHnd)) - { - JITDUMP("bailing, have shared instance type\n"); - return nullptr; - } - - // Simulate removing the box for thisOP. We need to know that it can - // be safely removed before we can optimize. - GenTree* thisVal = gtTryRemoveBoxUpstreamEffects(thisOp, BR_DONT_REMOVE); - if (thisVal == nullptr) - { - // Note we may fail here if the this operand comes from - // a call. We should be able to retry this post-inlining. - JITDUMP("bailing, can't undo box of 'this' operand\n"); - return nullptr; - } - - // Do likewise with flagOp. - GenTree* flagVal = gtTryRemoveBoxUpstreamEffects(flagOp, BR_DONT_REMOVE); - if (flagVal == nullptr) - { - // Note we may fail here if the flag operand comes from - // a call. We should be able to retry this post-inlining. - JITDUMP("bailing, can't undo box of 'flag' operand\n"); - return nullptr; - } - - // Only proceed when both box sources have the same actual type. - // (this rules out long/int mismatches) - if (genActualType(thisVal->TypeGet()) != genActualType(flagVal->TypeGet())) - { - JITDUMP("bailing, pre-boxed values have different types\n"); - return nullptr; - } - - // Yes, both boxes can be cleaned up. Optimize. - JITDUMP("Optimizing call to Enum.HasFlag\n"); - - // Undo the boxing of the Ops and prepare to operate directly - // on the pre-boxed values. - thisVal = gtTryRemoveBoxUpstreamEffects(thisOp, BR_REMOVE_BUT_NOT_NARROW); - flagVal = gtTryRemoveBoxUpstreamEffects(flagOp, BR_REMOVE_BUT_NOT_NARROW); - - // Our trial removals above should guarantee successful removals here. - assert(thisVal != nullptr); - assert(flagVal != nullptr); - assert(genActualType(thisVal->TypeGet()) == genActualType(flagVal->TypeGet())); - - // Type to use for optimized check - var_types type = genActualType(thisVal->TypeGet()); - - // The thisVal and flagVal trees come from earlier statements. - // - // Unless they are invariant values, we need to evaluate them both - // to temps at those points to safely transmit the values here. - // - // Also we need to use the flag twice, so we need two trees for it. - GenTree* thisValOpt = nullptr; - GenTree* flagValOpt = nullptr; - GenTree* flagValOptCopy = nullptr; - - if (thisVal->IsIntegralConst()) - { - thisValOpt = gtClone(thisVal); - assert(thisValOpt != nullptr); - } - else - { - const unsigned thisTmp = lvaGrabTemp(true DEBUGARG("Enum:HasFlag this temp")); - GenTree* thisStore = gtNewTempStore(thisTmp, thisVal); - Statement* thisStoreStmt = thisOp->AsBox()->gtCopyStmtWhenInlinedBoxValue; - thisStoreStmt->SetRootNode(thisStore); - thisValOpt = gtNewLclvNode(thisTmp, type); - - // If this is invoked during global morph we are adding code to a remote tree - // Despite this being a store, we can't meaningfully add assertions - // - thisStore->SetMorphed(this); - } - - if (flagVal->IsIntegralConst()) - { - flagValOpt = gtClone(flagVal); - assert(flagValOpt != nullptr); - flagValOptCopy = gtClone(flagVal); - assert(flagValOptCopy != nullptr); - } - else - { - const unsigned flagTmp = lvaGrabTemp(true DEBUGARG("Enum:HasFlag flag temp")); - GenTree* flagStore = gtNewTempStore(flagTmp, flagVal); - Statement* flagStoreStmt = flagOp->AsBox()->gtCopyStmtWhenInlinedBoxValue; - flagStoreStmt->SetRootNode(flagStore); - flagValOpt = gtNewLclvNode(flagTmp, type); - flagValOptCopy = gtNewLclvNode(flagTmp, type); - - // If this is invoked during global morph we are adding code to a remote tree - // Despite this being a store, we can't meaningfully add assertions - // - flagStore->SetMorphed(this); - } - - // Turn the call into (thisValTmp & flagTmp) == flagTmp. - GenTree* andTree = gtNewOperNode(GT_AND, type, thisValOpt, flagValOpt); - GenTree* cmpTree = gtNewOperNode(GT_EQ, TYP_INT, andTree, flagValOptCopy); - - JITDUMP("Optimized call to Enum.HasFlag\n"); - - return cmpTree; -} - -/***************************************************************************** - * - * Fold the given constant tree. - */ - -#ifdef _PREFAST_ -#pragma warning(push) -#pragma warning(disable : 21000) // Suppress PREFast warning about overly large function -#endif -GenTree* Compiler::gtFoldExprConst(GenTree* tree) -{ - SSIZE_T i1, i2, itemp; - INT64 lval1, lval2, ltemp; - float f1, f2; - double d1, d2; - var_types switchType; - FieldSeq* fieldSeq = nullptr; // default unless we override it when folding - - assert(tree->OperIsUnary() || tree->OperIsBinary()); - - GenTree* op1 = tree->gtGetOp1(); - GenTree* op2 = tree->gtGetOp2IfPresent(); - - if (!opts.Tier0OptimizationEnabled()) - { - return tree; - } - - if (tree->OperIs(GT_NOP, GT_ALLOCOBJ, GT_RUNTIMELOOKUP) || tree->OperIsStore() || tree->OperIsHWIntrinsic()) - { - return tree; - } - - if (tree->OperIsUnary()) - { - assert(op1->OperIsConst()); - - switch (op1->TypeGet()) - { - case TYP_INT: - - // Fold constant INT unary operator. - - if (!op1->AsIntCon()->ImmedValCanBeFolded(this, tree->OperGet())) - { - return tree; - } - - i1 = (INT32)op1->AsIntCon()->IconValue(); - - switch (tree->OperGet()) - { - case GT_NOT: - i1 = ~i1; - break; - - case GT_NEG: - i1 = -i1; - break; - - case GT_BSWAP: - i1 = ((i1 >> 24) & 0xFF) | ((i1 >> 8) & 0xFF00) | ((i1 << 8) & 0xFF0000) | - ((i1 << 24) & 0xFF000000); - break; - - case GT_BSWAP16: - i1 = ((i1 >> 8) & 0xFF) | ((i1 << 8) & 0xFF00); - break; - - case GT_CAST: - // assert (genActualType(tree->CastToType()) == tree->TypeGet()); - - if (tree->gtOverflow() && - CheckedOps::CastFromIntOverflows((INT32)i1, tree->CastToType(), tree->IsUnsigned())) - { - goto INTEGRAL_OVF; - } - - switch (tree->CastToType()) - { - case TYP_BYTE: - i1 = INT32(INT8(i1)); - goto CNS_INT; - - case TYP_SHORT: - i1 = INT32(INT16(i1)); - goto CNS_INT; - - case TYP_USHORT: - i1 = INT32(UINT16(i1)); - goto CNS_INT; - - case TYP_UBYTE: - i1 = INT32(UINT8(i1)); - goto CNS_INT; - - case TYP_UINT: - case TYP_INT: - goto CNS_INT; - - case TYP_ULONG: - if (tree->IsUnsigned()) - { - lval1 = UINT64(UINT32(i1)); - } - else - { - lval1 = UINT64(INT32(i1)); - } - goto CNS_LONG; - - case TYP_LONG: - if (tree->IsUnsigned()) - { - lval1 = INT64(UINT32(i1)); - } - else - { - lval1 = INT64(INT32(i1)); - } - goto CNS_LONG; - - case TYP_FLOAT: - { -#if defined(TARGET_64BIT) - if (tree->IsUnsigned()) - { - f1 = (float)UINT32(i1); - } - else - { - f1 = (float)INT32(i1); - } -#else - // 32-bit currently does a 2-step conversion, which is incorrect - // but which we are going to take a breaking change around early - // in a release cycle. - - if (tree->IsUnsigned()) - { - f1 = forceCastToFloat(UINT32(i1)); - } - else - { - f1 = forceCastToFloat(INT32(i1)); - } -#endif - - d1 = f1; - goto CNS_DOUBLE; - } - - case TYP_DOUBLE: - { - if (tree->IsUnsigned()) - { - d1 = (double)UINT32(i1); - } - else - { - d1 = (double)INT32(i1); - } - goto CNS_DOUBLE; - } - - default: - assert(!"Bad CastToType() in gtFoldExprConst() for a cast from int"); - return tree; - } - - default: - return tree; - } - - goto CNS_INT; - - case TYP_LONG: - - // Fold constant LONG unary operator. - - if (!op1->AsIntConCommon()->ImmedValCanBeFolded(this, tree->OperGet())) - { - return tree; - } - - lval1 = op1->AsIntConCommon()->LngValue(); - - switch (tree->OperGet()) - { - case GT_NOT: - lval1 = ~lval1; - break; - - case GT_NEG: - lval1 = -lval1; - break; - - case GT_BSWAP: - lval1 = ((lval1 >> 56) & 0xFF) | ((lval1 >> 40) & 0xFF00) | ((lval1 >> 24) & 0xFF0000) | - ((lval1 >> 8) & 0xFF000000) | ((lval1 << 8) & 0xFF00000000) | - ((lval1 << 24) & 0xFF0000000000) | ((lval1 << 40) & 0xFF000000000000) | - ((lval1 << 56) & 0xFF00000000000000); - break; - - case GT_CAST: - assert(tree->TypeIs(genActualType(tree->CastToType()))); - - if (tree->gtOverflow() && - CheckedOps::CastFromLongOverflows(lval1, tree->CastToType(), tree->IsUnsigned())) - { - goto INTEGRAL_OVF; - } - - switch (tree->CastToType()) - { - case TYP_BYTE: - i1 = INT32(INT8(lval1)); - goto CNS_INT; - - case TYP_SHORT: - i1 = INT32(INT16(lval1)); - goto CNS_INT; - - case TYP_USHORT: - i1 = INT32(UINT16(lval1)); - goto CNS_INT; - - case TYP_UBYTE: - i1 = INT32(UINT8(lval1)); - goto CNS_INT; - - case TYP_INT: - i1 = INT32(lval1); - goto CNS_INT; - - case TYP_UINT: - i1 = UINT32(lval1); - goto CNS_INT; - - case TYP_ULONG: - case TYP_LONG: - goto CNS_LONG; - - case TYP_FLOAT: - { -#ifdef TARGET_64BIT - if (tree->IsUnsigned() && (lval1 < 0)) - { - f1 = FloatingPointUtils::convertUInt64ToFloat((uint64_t)lval1); - } - else - { - f1 = (float)lval1; - } -#else - // 32-bit currently does a 2-step conversion, which is incorrect - // but which we are going to take a breaking change around early - // in a release cycle. - - if (tree->IsUnsigned() && (lval1 < 0)) - { - f1 = forceCastToFloat(FloatingPointUtils::convertUInt64ToDouble((uint64_t)lval1)); - } - else - { - f1 = forceCastToFloat((double)lval1); - } -#endif - - d1 = f1; - goto CNS_DOUBLE; - } - - case TYP_DOUBLE: - { - if (tree->IsUnsigned() && (lval1 < 0)) - { - d1 = FloatingPointUtils::convertUInt64ToDouble((uint64_t)lval1); - } - else - { - d1 = (double)lval1; - } - goto CNS_DOUBLE; - } - default: - assert(!"Bad CastToType() in gtFoldExprConst() for a cast from long"); - return tree; - } - - default: - return tree; - } - - goto CNS_LONG; - - case TYP_FLOAT: - case TYP_DOUBLE: - assert(op1->OperIs(GT_CNS_DBL)); - - // Fold constant DOUBLE unary operator. - - d1 = op1->AsDblCon()->DconValue(); - - switch (tree->OperGet()) - { - case GT_NEG: - d1 = -d1; - break; - - case GT_CAST: - f1 = forceCastToFloat(d1); - - if ((op1->TypeIs(TYP_DOUBLE) && CheckedOps::CastFromDoubleOverflows(d1, tree->CastToType())) || - (op1->TypeIs(TYP_FLOAT) && CheckedOps::CastFromFloatOverflows(f1, tree->CastToType()))) - { - // The conversion overflows. The ECMA spec says, in III 3.27, that - // "...if overflow occurs converting a floating point type to an integer, ..., - // the value returned is unspecified." However, it would at least be - // desirable to have the same value returned for casting an overflowing - // constant to an int as would be obtained by passing that constant as - // a parameter and then casting that parameter to an int type. - - // Don't fold overflowing conversions, as the value returned by - // JIT's codegen doesn't always match with the C compiler's cast result. - // We want the behavior to be the same with or without folding. - - return tree; - } - - assert(tree->TypeIs(genActualType(tree->CastToType()))); - - switch (tree->CastToType()) - { - case TYP_BYTE: - i1 = INT32(INT8(d1)); - goto CNS_INT; - - case TYP_SHORT: - i1 = INT32(INT16(d1)); - goto CNS_INT; - - case TYP_USHORT: - i1 = INT32(UINT16(d1)); - goto CNS_INT; - - case TYP_UBYTE: - i1 = INT32(UINT8(d1)); - goto CNS_INT; - - case TYP_INT: - i1 = INT32(d1); - goto CNS_INT; - - case TYP_UINT: - i1 = forceCastToUInt32(d1); - goto CNS_INT; - - case TYP_LONG: - lval1 = INT64(d1); - goto CNS_LONG; - - case TYP_ULONG: - lval1 = FloatingPointUtils::convertDoubleToUInt64(d1); - goto CNS_LONG; - - case TYP_FLOAT: - d1 = forceCastToFloat(d1); - goto CNS_DOUBLE; - - case TYP_DOUBLE: - if (op1->TypeIs(TYP_FLOAT)) - { - d1 = forceCastToFloat(d1); // Truncate precision. - } - goto CNS_DOUBLE; // Redundant cast. - - default: - assert(!"Bad CastToType() in gtFoldExprConst() for a cast from double/float"); - break; - } - return tree; - - default: - return tree; - } - goto CNS_DOUBLE; - - default: - // Not a foldable typ - e.g. RET const. - return tree; - } - } - - // We have a binary operator. - - assert(tree->OperIsBinary()); - assert(op2 != nullptr); - assert(op1->OperIsConst()); - assert(op2->OperIsConst()); - - if (tree->OperIs(GT_COMMA)) - { - return op2; - } - - if (tree->OperIs(GT_BOUNDS_CHECK)) - { - ssize_t index = op1->AsIntCon()->IconValue(); - ssize_t length = op2->AsIntCon()->IconValue(); - - if ((0 <= index) && (index < length)) - { - JITDUMP("\nFolding an in-range bounds check:\n"); - DISPTREE(tree); - - tree->gtBashToNOP(); - - JITDUMP("Bashed to NOP:\n"); - DISPTREE(tree); - } - - return tree; - } - - switchType = op1->TypeGet(); - - // Normally we will just switch on op1 types, but for the case where - // only op2 is a GC type and op1 is not a GC type, we use the op2 type. - // This makes us handle this as a case of folding for GC type. - if (varTypeIsGC(op2->gtType) && !varTypeIsGC(op1->gtType)) - { - switchType = op2->TypeGet(); - } - - switch (switchType) - { - // Fold constant REF of BYREF binary operator. - // These can only be comparisons or null pointers. - - case TYP_REF: - - // String nodes are an RVA at this point. - if (op1->OperIs(GT_CNS_STR) || op2->OperIs(GT_CNS_STR)) - { - // Fold "ldstr" ==/!= null. - if (op2->IsIntegralConst(0)) - { - if (tree->OperIs(GT_EQ)) - { - i1 = 0; - goto FOLD_COND; - } - if (tree->OperIs(GT_NE) || (tree->OperIs(GT_GT) && tree->IsUnsigned())) - { - i1 = 1; - goto FOLD_COND; - } - } - return tree; - } - - FALLTHROUGH; - - case TYP_BYREF: - - i1 = op1->AsIntConCommon()->IconValue(); - i2 = op2->AsIntConCommon()->IconValue(); - - switch (tree->OperGet()) - { - case GT_EQ: - i1 = (i1 == i2); - goto FOLD_COND; - - case GT_NE: - i1 = (i1 != i2); - goto FOLD_COND; - - case GT_ADD: - noway_assert(!tree->TypeIs(TYP_REF)); - // We only fold a GT_ADD that involves a null reference. - if ((op1->TypeIs(TYP_REF) && (i1 == 0)) || (op2->TypeIs(TYP_REF) && (i2 == 0))) - { - JITDUMP("\nFolding operator with constant nodes into a constant:\n"); - DISPTREE(tree); - - // Fold into GT_IND of null byref. - tree->BashToConst(0, TYP_BYREF); - fgUpdateConstTreeValueNumber(tree); - - JITDUMP("\nFolded to null byref:\n"); - DISPTREE(tree); - - goto DONE; - } - break; - - default: - break; - } - - return tree; - - // Fold constant INT binary operator. - - case TYP_INT: - - assert(tree->TypeIs(TYP_INT) || varTypeIsGC(tree)); - // No GC pointer types should be folded here... - assert(!varTypeIsGC(op1->TypeGet()) && !varTypeIsGC(op2->TypeGet())); - - if (!op1->AsIntConCommon()->ImmedValCanBeFolded(this, tree->OperGet())) - { - return tree; - } - - if (!op2->AsIntConCommon()->ImmedValCanBeFolded(this, tree->OperGet())) - { - return tree; - } - - i1 = op1->AsIntConCommon()->IconValue(); - i2 = op2->AsIntConCommon()->IconValue(); - - switch (tree->OperGet()) - { - case GT_EQ: - i1 = (INT32(i1) == INT32(i2)); - break; - case GT_NE: - i1 = (INT32(i1) != INT32(i2)); - break; - - case GT_LT: - if (tree->IsUnsigned()) - { - i1 = (UINT32(i1) < UINT32(i2)); - } - else - { - i1 = (INT32(i1) < INT32(i2)); - } - break; - - case GT_LE: - if (tree->IsUnsigned()) - { - i1 = (UINT32(i1) <= UINT32(i2)); - } - else - { - i1 = (INT32(i1) <= INT32(i2)); - } - break; - - case GT_GE: - if (tree->IsUnsigned()) - { - i1 = (UINT32(i1) >= UINT32(i2)); - } - else - { - i1 = (INT32(i1) >= INT32(i2)); - } - break; - - case GT_GT: - if (tree->IsUnsigned()) - { - i1 = (UINT32(i1) > UINT32(i2)); - } - else - { - i1 = (INT32(i1) > INT32(i2)); - } - break; - - case GT_ADD: - itemp = i1 + i2; - if (tree->gtOverflow() && CheckedOps::AddOverflows(INT32(i1), INT32(i2), tree->IsUnsigned())) - { - goto INTEGRAL_OVF; - } - i1 = itemp; - fieldSeq = GetFieldSeqStore()->Append(op1->AsIntCon()->gtFieldSeq, op2->AsIntCon()->gtFieldSeq); - break; - case GT_SUB: - itemp = i1 - i2; - if (tree->gtOverflow() && CheckedOps::SubOverflows(INT32(i1), INT32(i2), tree->IsUnsigned())) - { - goto INTEGRAL_OVF; - } - i1 = itemp; - break; - case GT_MUL: - itemp = i1 * i2; - if (tree->gtOverflow() && CheckedOps::MulOverflows(INT32(i1), INT32(i2), tree->IsUnsigned())) - { - goto INTEGRAL_OVF; - } - i1 = itemp; - break; - - case GT_OR: - i1 |= i2; - break; - case GT_XOR: - i1 ^= i2; - break; - case GT_AND: - i1 &= i2; - break; - - case GT_LSH: - i1 <<= (i2 & 0x1f); - break; - case GT_RSH: - i1 >>= (i2 & 0x1f); - break; - case GT_RSZ: - // logical shift -> make it unsigned to not propagate the sign bit. - i1 = UINT32(i1) >> (i2 & 0x1f); - break; - case GT_ROL: - i1 = (i1 << (i2 & 0x1f)) | (UINT32(i1) >> ((32 - i2) & 0x1f)); - break; - case GT_ROR: - i1 = (i1 << ((32 - i2) & 0x1f)) | (UINT32(i1) >> (i2 & 0x1f)); - break; - - // DIV and MOD can throw an exception - if the division is by 0 - // or there is overflow - when dividing MIN by -1. - - case GT_DIV: - case GT_MOD: - case GT_UDIV: - case GT_UMOD: - if (INT32(i2) == 0) - { - // Division by zero. - // We have to evaluate this expression and throw an exception. - return tree; - } - else if ((INT32(i2) == -1) && (UINT32(i1) == 0x80000000)) - { - // Overflow Division. - // We have to evaluate this expression and throw an exception. - return tree; - } - - if (tree->OperIs(GT_DIV)) - { - i1 = INT32(i1) / INT32(i2); - } - else if (tree->OperIs(GT_MOD)) - { - i1 = INT32(i1) % INT32(i2); - } - else if (tree->OperIs(GT_UDIV)) - { - i1 = UINT32(i1) / UINT32(i2); - } - else - { - assert(tree->OperIs(GT_UMOD)); - i1 = UINT32(i1) % UINT32(i2); - } - break; - - default: - return tree; - } - - // We get here after folding to a GT_CNS_INT type. - // change the node to the new type / value and make sure the node sizes are OK. - CNS_INT: - FOLD_COND: - - JITDUMP("\nFolding operator with constant nodes into a constant:\n"); - DISPTREE(tree); - - // Also all conditional folding jumps here since the node hanging from - // GT_JTRUE has to be a GT_CNS_INT - value 0 or 1. - - // Some operations are performed as 64 bit instead of 32 bit so the upper 32 bits - // need to be discarded. Since constant values are stored as ssize_t and the node - // has TYP_INT the result needs to be sign extended rather than zero extended. - tree->BashToConst(static_cast(i1)); - tree->AsIntCon()->gtFieldSeq = fieldSeq; - - fgUpdateConstTreeValueNumber(tree); - - JITDUMP("Bashed to int constant:\n"); - DISPTREE(tree); - - goto DONE; - - // Fold constant LONG binary operator. - - case TYP_LONG: - - // No GC pointer types should be folded here... - assert(!varTypeIsGC(op1->TypeGet()) && !varTypeIsGC(op2->TypeGet())); - - // op1 is known to be a TYP_LONG, op2 is normally a TYP_LONG, unless we have a shift operator in which case - // it is a TYP_INT. - assert(op2->TypeIs(TYP_LONG, TYP_INT)); - - if (!op1->AsIntConCommon()->ImmedValCanBeFolded(this, tree->OperGet())) - { - return tree; - } - - if (!op2->AsIntConCommon()->ImmedValCanBeFolded(this, tree->OperGet())) - { - return tree; - } - - lval1 = op1->AsIntConCommon()->LngValue(); - - // For the shift operators we can have a op2 that is a TYP_INT. - // Thus we cannot just use LngValue(), as it will assert on 32 bit if op2 is not GT_CNS_LNG. - lval2 = op2->AsIntConCommon()->IntegralValue(); - - switch (tree->OperGet()) - { - case GT_EQ: - i1 = (lval1 == lval2); - goto FOLD_COND; - case GT_NE: - i1 = (lval1 != lval2); - goto FOLD_COND; - - case GT_LT: - if (tree->IsUnsigned()) - { - i1 = (UINT64(lval1) < UINT64(lval2)); - } - else - { - i1 = (lval1 < lval2); - } - goto FOLD_COND; - - case GT_LE: - if (tree->IsUnsigned()) - { - i1 = (UINT64(lval1) <= UINT64(lval2)); - } - else - { - i1 = (lval1 <= lval2); - } - goto FOLD_COND; - - case GT_GE: - if (tree->IsUnsigned()) - { - i1 = (UINT64(lval1) >= UINT64(lval2)); - } - else - { - i1 = (lval1 >= lval2); - } - goto FOLD_COND; - - case GT_GT: - if (tree->IsUnsigned()) - { - i1 = (UINT64(lval1) > UINT64(lval2)); - } - else - { - i1 = (lval1 > lval2); - } - goto FOLD_COND; - - case GT_ADD: - ltemp = lval1 + lval2; - if (tree->gtOverflow() && CheckedOps::AddOverflows(lval1, lval2, tree->IsUnsigned())) - { - goto INTEGRAL_OVF; - } - lval1 = ltemp; -#ifdef TARGET_64BIT - fieldSeq = GetFieldSeqStore()->Append(op1->AsIntCon()->gtFieldSeq, op2->AsIntCon()->gtFieldSeq); -#endif - break; - - case GT_SUB: - ltemp = lval1 - lval2; - if (tree->gtOverflow() && CheckedOps::SubOverflows(lval1, lval2, tree->IsUnsigned())) - { - goto INTEGRAL_OVF; - } - lval1 = ltemp; - break; - - case GT_MUL: - ltemp = lval1 * lval2; - if (tree->gtOverflow() && CheckedOps::MulOverflows(lval1, lval2, tree->IsUnsigned())) - { - goto INTEGRAL_OVF; - } - lval1 = ltemp; - break; - - case GT_OR: - lval1 |= lval2; - break; - case GT_XOR: - lval1 ^= lval2; - break; - case GT_AND: - lval1 &= lval2; - break; - - case GT_LSH: - lval1 <<= (lval2 & 0x3f); - break; - case GT_RSH: - lval1 >>= (lval2 & 0x3f); - break; - case GT_RSZ: - // logical shift -> make it unsigned to not propagate the sign bit. - lval1 = UINT64(lval1) >> (lval2 & 0x3f); - break; - case GT_ROL: - lval1 = (lval1 << (lval2 & 0x3f)) | (UINT64(lval1) >> ((64 - lval2) & 0x3f)); - break; - case GT_ROR: - lval1 = (lval1 << ((64 - lval2) & 0x3f)) | (UINT64(lval1) >> (lval2 & 0x3f)); - break; - - // Both DIV and IDIV on x86 raise an exception for min_int (and min_long) / -1. So we preserve - // that behavior here. - case GT_DIV: - if (lval2 == 0) - { - return tree; - } - if ((UINT64(lval1) == UINT64(0x8000000000000000)) && (lval2 == INT64(-1))) - { - return tree; - } - - lval1 /= lval2; - break; - - case GT_MOD: - if (lval2 == 0) - { - return tree; - } - if ((UINT64(lval1) == UINT64(0x8000000000000000)) && (lval2 == INT64(-1))) - { - return tree; - } - - lval1 %= lval2; - break; - - case GT_UDIV: - if (lval2 == 0) - { - return tree; - } - if ((UINT64(lval1) == UINT64(0x8000000000000000)) && (lval2 == INT64(-1))) - { - return tree; - } - - lval1 = UINT64(lval1) / UINT64(lval2); - break; - - case GT_UMOD: - if (lval2 == 0) - { - return tree; - } - if ((UINT64(lval1) == UINT64(0x8000000000000000)) && (lval2 == INT64(-1))) - { - return tree; - } - - lval1 = UINT64(lval1) % UINT64(lval2); - break; - default: - return tree; - } - - CNS_LONG: -#if !defined(TARGET_64BIT) - if (fieldSeq != nullptr) - { - assert(!"Field sequences on CNS_LNG nodes!?"); - return tree; - } -#endif // !defined(TARGET_64BIT) - - JITDUMP("\nFolding long operator with constant nodes into a constant:\n"); - DISPTREE(tree); - - assert((GenTree::s_gtNodeSizes[GT_CNS_NATIVELONG] == TREE_NODE_SZ_SMALL) || - (tree->gtDebugFlags & GTF_DEBUG_NODE_LARGE)); - - tree->BashToConst(lval1); -#ifdef TARGET_64BIT - tree->AsIntCon()->gtFieldSeq = fieldSeq; -#endif - fgUpdateConstTreeValueNumber(tree); - - JITDUMP("Bashed to long constant:\n"); - DISPTREE(tree); - - goto DONE; - - // Fold constant FLOAT or DOUBLE binary operator - - case TYP_FLOAT: - case TYP_DOUBLE: - - if (tree->gtOverflowEx()) - { - return tree; - } - - assert(op1->OperIs(GT_CNS_DBL)); - d1 = op1->AsDblCon()->DconValue(); - - assert(varTypeIsFloating(op2->TypeGet())); - assert(op2->OperIs(GT_CNS_DBL)); - d2 = op2->AsDblCon()->DconValue(); - - // Special case - check if we have NaN operands. - // For comparisons if not an unordered operation always return 0. - // For unordered operations (i.e. the GTF_RELOP_NAN_UN flag is set) - // the result is always true - return 1. - - if (FloatingPointUtils::isNaN(d1) || FloatingPointUtils::isNaN(d2)) - { - JITDUMP("Double operator(s) is NaN\n"); - - if (tree->OperIsCompare()) - { - if (tree->gtFlags & GTF_RELOP_NAN_UN) - { - // Unordered comparison with NaN always succeeds. - i1 = 1; - goto FOLD_COND; - } - else - { - // Normal comparison with NaN always fails. - i1 = 0; - goto FOLD_COND; - } - } - } - - switch (tree->OperGet()) - { - case GT_EQ: - i1 = (d1 == d2); - goto FOLD_COND; - case GT_NE: - i1 = (d1 != d2); - goto FOLD_COND; - - case GT_LT: - i1 = (d1 < d2); - goto FOLD_COND; - case GT_LE: - i1 = (d1 <= d2); - goto FOLD_COND; - case GT_GE: - i1 = (d1 >= d2); - goto FOLD_COND; - case GT_GT: - i1 = (d1 > d2); - goto FOLD_COND; - - // Floating point arithmetic should be done in declared - // precision while doing constant folding. For this reason though TYP_FLOAT - // constants are stored as double constants, while performing float arithmetic, - // double constants should be converted to float. Here is an example case - // where performing arithmetic in double precision would lead to incorrect - // results. - // - // Example: - // float a = float.MaxValue; - // float b = a*a; This will produce +inf in single precision and 1.1579207543382391e+077 in double - // precision. - // float c = b/b; This will produce NaN in single precision and 1 in double precision. - case GT_ADD: - if (op1->TypeIs(TYP_FLOAT)) - { - f1 = forceCastToFloat(d1); - f2 = forceCastToFloat(d2); - d1 = forceCastToFloat(f1 + f2); - } - else - { - d1 += d2; - } - break; - - case GT_SUB: - if (op1->TypeIs(TYP_FLOAT)) - { - f1 = forceCastToFloat(d1); - f2 = forceCastToFloat(d2); - d1 = forceCastToFloat(f1 - f2); - } - else - { - d1 -= d2; - } - break; - - case GT_MUL: - if (op1->TypeIs(TYP_FLOAT)) - { - f1 = forceCastToFloat(d1); - f2 = forceCastToFloat(d2); - d1 = forceCastToFloat(f1 * f2); - } - else - { - d1 *= d2; - } - break; - - case GT_DIV: - // We do not fold division by zero, even for floating point. - // This is because the result will be platform-dependent for an expression like 0d / 0d. - if (d2 == 0) - { - return tree; - } - if (op1->TypeIs(TYP_FLOAT)) - { - f1 = forceCastToFloat(d1); - f2 = forceCastToFloat(d2); - d1 = forceCastToFloat(f1 / f2); - } - else - { - d1 /= d2; - } - break; - - default: - return tree; - } - - CNS_DOUBLE: - - JITDUMP("\nFolding fp operator with constant nodes into a fp constant:\n"); - DISPTREE(tree); - - assert((GenTree::s_gtNodeSizes[GT_CNS_DBL] == TREE_NODE_SZ_SMALL) || - (tree->gtDebugFlags & GTF_DEBUG_NODE_LARGE)); - - tree->BashToConst(d1, tree->TypeGet()); - fgUpdateConstTreeValueNumber(tree); - - JITDUMP("Bashed to fp constant:\n"); - DISPTREE(tree); - - goto DONE; - - default: - // Not a foldable type. - return tree; - } - -DONE: - - // Make sure no side effect flags are set on this constant node. - - tree->gtFlags &= ~GTF_ALL_EFFECT; - - return tree; - -INTEGRAL_OVF: - - // This operation is going to cause an overflow exception. Morph into - // an overflow helper. Put a dummy constant value for code generation. - // - // We could remove all subsequent trees in the current basic block, - // unless this node is a child of GT_COLON - // - // NOTE: Since the folded value is not constant we should not change the - // "tree" node - otherwise we confuse the logic that checks if the folding - // was successful - instead use one of the operands, e.g. op1. - - // Don't fold overflow operations if not global morph phase. - // The reason for this is that this optimization is replacing a GenTree node - // with another new GenTree node. Say a GT_CALL(arglist) has one 'arg' - // involving overflow arithmetic. During assertion prop, it is possible - // that the 'arg' could be constant folded and the result could lead to an - // overflow. In such a case 'arg' will get replaced with GT_COMMA node - // but fgMorphArgs() - see the logic around "if(lateArgsComputed)" - doesn't - // update args table. For this reason this optimization is enabled only - // for global morphing phase. - // - // TODO-CQ: Once fgMorphArgs() is fixed this restriction could be removed. - - if (!fgGlobalMorph) - { - assert(tree->gtOverflow()); - return tree; - } - - var_types type = genActualType(tree->TypeGet()); - op1 = type == TYP_LONG ? gtNewLconNode(0) : gtNewIconNode(0); - if (vnStore != nullptr) - { - op1->gtVNPair.SetBoth(vnStore->VNZeroForType(type)); - } - - JITDUMP("\nFolding binary operator with constant nodes into a comma throw:\n"); - DISPTREE(tree); - - // We will change the cast to a GT_COMMA and attach the exception helper as AsOp()->gtOp1. - // The constant expression zero becomes op2. - - assert(tree->gtOverflow()); - assert(tree->OperIs(GT_ADD, GT_SUB, GT_CAST, GT_MUL)); - assert(op1 != nullptr); - - op2 = op1; - op1 = gtNewHelperCallNode(CORINFO_HELP_OVERFLOW, TYP_VOID, gtNewIconNode(compCurBB->bbTryIndex)); - - // op1 is a call to the JIT helper that throws an Overflow exception. - // Attach the ExcSet for VNF_OverflowExc(Void) to this call. - - if (vnStore != nullptr) - { - op1->gtVNPair = vnStore->VNPWithExc(ValueNumPair(ValueNumStore::VNForVoid(), ValueNumStore::VNForVoid()), - vnStore->VNPExcSetSingleton(vnStore->VNPairForFunc(TYP_REF, VNF_OverflowExc, - vnStore->VNPForVoid()))); - } - - tree = gtNewOperNode(GT_COMMA, tree->TypeGet(), op1, op2); - - return tree; -} -#ifdef _PREFAST_ -#pragma warning(pop) -#endif - -//------------------------------------------------------------------------ -// gtFoldIndirConst: Attempt to fold an "IND(addr)" expression to a constant. -// -// Currently handles the case of "addr" being "INDEX_ADDR(CNS_STR, CONST)". -// -// Arguments: -// indir - The IND node to attempt to fold -// -// Return Value: -// The new constant node if the folding was successful, "nullptr" otherwise. -// -GenTree* Compiler::gtFoldIndirConst(GenTreeIndir* indir) -{ - assert(opts.OptimizationEnabled() && !optValnumCSE_phase); - assert(indir->OperIs(GT_IND)); - - GenTree* addr = indir->Addr(); - - if (indir->TypeIs(TYP_USHORT) && addr->OperIs(GT_INDEX_ADDR) && addr->AsIndexAddr()->Arr()->OperIs(GT_CNS_STR)) - { - GenTreeStrCon* stringNode = addr->AsIndexAddr()->Arr()->AsStrCon(); - GenTree* indexNode = addr->AsIndexAddr()->Index(); - if (!stringNode->IsStringEmptyField() && indexNode->IsCnsIntOrI()) - { - int cnsIndex = static_cast(indexNode->AsIntConCommon()->IconValue()); - if (cnsIndex >= 0) - { - char16_t chr; - int length = - info.compCompHnd->getStringLiteral(stringNode->gtScpHnd, stringNode->gtSconCPX, &chr, 1, cnsIndex); - if (length > 0) - { - return gtNewIconNode(chr); - } - } - } - } - - return nullptr; -} - -//------------------------------------------------------------------------ -// gtNewTempStore: Create a store of the given value to a temp. -// -// Arguments: -// tmp - local number for a compiler temp -// val - value to store to the temp -// curLevel - stack level to spill at (importer-only) -// pAfterStmt - statement to insert any additional statements after -// di - debug info for new statements -// block - block to insert any additional statements in -// -// Return Value: -// Normally a new store node. -// However may return a nop node if val is simply a reference to the temp. -// -// Notes: -// Self-stores may be represented via NOPs. -// May update the type of the temp, if it was previously unknown. -// May set compFloatingPointUsed. -// -GenTree* Compiler::gtNewTempStore( - unsigned tmp, GenTree* val, unsigned curLevel, Statement** pAfterStmt, const DebugInfo& di, BasicBlock* block) -{ - // Self-assignment is a nop. - if (val->OperGet() == GT_LCL_VAR && val->AsLclVarCommon()->GetLclNum() == tmp) - { - return gtNewNothingNode(); - } - - LclVarDsc* varDsc = lvaGetDesc(tmp); - - if (varDsc->TypeGet() == TYP_I_IMPL && val->TypeGet() == TYP_BYREF) - { - impBashVarAddrsToI(val); - } - - var_types valTyp = val->TypeGet(); - if (val->OperGet() == GT_LCL_VAR && lvaTable[val->AsLclVar()->GetLclNum()].lvNormalizeOnLoad()) - { - valTyp = lvaGetRealType(val->AsLclVar()->GetLclNum()); - val->gtType = valTyp; - } - - var_types dstTyp = varDsc->TypeGet(); - - /* If the variable's lvType is not yet set then set it here */ - if (dstTyp == TYP_UNDEF) - { - varDsc->lvType = dstTyp = genActualType(valTyp); - - if (dstTyp == TYP_STRUCT) - { - lvaSetStruct(tmp, val->GetLayout(this), false); - } - } - -#ifdef DEBUG - // Make sure the actual types match. - if (genActualType(valTyp) != genActualType(dstTyp)) - { - // Plus some other exceptions that are apparently legal: - // - TYP_REF or BYREF = TYP_I_IMPL - bool ok = false; - if (varTypeIsGC(dstTyp) && (valTyp == TYP_I_IMPL)) - { - ok = true; - } - // - TYP_I_IMPL = TYP_BYREF - else if ((dstTyp == TYP_I_IMPL) && (valTyp == TYP_BYREF)) - { - ok = true; - } - // - TYP_BYREF = TYP_REF when object stack allocation is enabled - else if (JitConfig.JitObjectStackAllocation() && (dstTyp == TYP_BYREF) && (valTyp == TYP_REF)) - { - ok = true; - } - else if ((dstTyp == TYP_STRUCT) && (valTyp == TYP_INT)) - { - assert(val->IsInitVal()); - ok = true; - } - - if (!ok) - { - gtDispTree(val); - assert(!"Incompatible types for gtNewTempStore"); - } - } -#endif - - // Added this noway_assert for runtime\issue 44895, to protect against silent bad codegen - // - if ((dstTyp == TYP_STRUCT) && (valTyp == TYP_REF)) - { - noway_assert(!"Incompatible types for gtNewTempStore"); - } - - // Floating Point stores can be created during inlining - // see "Zero init inlinee locals:" in fgInlinePrependStatements - // thus we may need to set compFloatingPointUsed to true here. - // - if (!varTypeUsesIntReg(dstTyp)) - { - compFloatingPointUsed = true; - } - - GenTree* store = gtNewStoreLclVarNode(tmp, val); - - // TODO-ASG: delete this zero-diff quirk. Requires some forward substitution work. - store->gtType = dstTyp; - - if (varTypeIsStruct(varDsc) && !val->IsInitVal()) - { - store = impStoreStruct(store, curLevel, pAfterStmt, di, block); - } - - return store; -} - -/***************************************************************************** - * - * Create a helper call to access a COM field (iff 'value' is non-zero this is - * a store and 'value' is the new value). - */ - -GenTree* Compiler::gtNewRefCOMfield(GenTree* objPtr, - CORINFO_RESOLVED_TOKEN* pResolvedToken, - CORINFO_ACCESS_FLAGS access, - CORINFO_FIELD_INFO* pFieldInfo, - var_types lclTyp, - GenTree* value) -{ - assert(pFieldInfo->fieldAccessor == CORINFO_FIELD_INSTANCE_HELPER || - pFieldInfo->fieldAccessor == CORINFO_FIELD_INSTANCE_ADDR_HELPER || - pFieldInfo->fieldAccessor == CORINFO_FIELD_STATIC_ADDR_HELPER); - - // Arguments in reverse order - GenTree* args[4]; - size_t nArgs = 0; - /* If we can't access it directly, we need to call a helper function */ - var_types helperType = TYP_BYREF; - CORINFO_CLASS_HANDLE structType = pFieldInfo->structType; - - if (pFieldInfo->fieldAccessor == CORINFO_FIELD_INSTANCE_HELPER) - { - if (access & CORINFO_ACCESS_SET) - { - assert(value != nullptr); - if ((lclTyp == TYP_DOUBLE) && (value->TypeGet() == TYP_FLOAT)) - { - value = gtNewCastNode(TYP_DOUBLE, value, false, TYP_DOUBLE); - } - else if (lclTyp == TYP_FLOAT && value->TypeGet() == TYP_DOUBLE) - { - value = gtNewCastNode(TYP_FLOAT, value, false, TYP_FLOAT); - } - - args[nArgs++] = value; - helperType = TYP_VOID; - } - else if (access & CORINFO_ACCESS_GET) - { - helperType = lclTyp; - } - } - - GenTree* fieldHnd = impTokenToHandle(pResolvedToken); - if (fieldHnd == nullptr) - { // compDonotInline() - return nullptr; - } - - args[nArgs++] = fieldHnd; - - // If it's a static field, we shouldn't have an object node - // If it's an instance field, we have an object node - assert((pFieldInfo->fieldAccessor != CORINFO_FIELD_STATIC_ADDR_HELPER) ^ (objPtr == nullptr)); - - if (objPtr != nullptr) - { - args[nArgs++] = objPtr; - } - - GenTreeCall* call = gtNewHelperCallNode(pFieldInfo->helper, genActualType(helperType)); - - for (size_t i = 0; i < nArgs; i++) - { - call->gtArgs.PushFront(this, NewCallArg::Primitive(args[i])); - call->gtFlags |= args[i]->gtFlags & GTF_ALL_EFFECT; - } - -#if FEATURE_MULTIREG_RET - if (varTypeIsStruct(call)) - { - call->InitializeStructReturnType(this, structType, call->GetUnmanagedCallConv()); - } -#endif // FEATURE_MULTIREG_RET - - GenTree* result = call; - - if (pFieldInfo->fieldAccessor == CORINFO_FIELD_INSTANCE_HELPER) - { - if (((access & CORINFO_ACCESS_GET) != 0) && varTypeIsSmall(lclTyp)) - { - // The helper does not extend the small return types. - result = gtNewCastNode(genActualType(lclTyp), result, false, lclTyp); - } - } - else if ((access & CORINFO_ACCESS_ADDRESS) == 0) // OK, now do the indirection - { - ClassLayout* layout; - lclTyp = TypeHandleToVarType(pFieldInfo->fieldType, structType, &layout); - - if ((access & CORINFO_ACCESS_SET) != 0) - { - result = (lclTyp == TYP_STRUCT) ? gtNewStoreBlkNode(layout, result, value)->AsIndir() - : gtNewStoreIndNode(lclTyp, result, value); - if (varTypeIsStruct(lclTyp)) - { - result = impStoreStruct(result, CHECK_SPILL_ALL); - } - } - else - { - assert((access & CORINFO_ACCESS_GET) != 0); - result = (lclTyp == TYP_STRUCT) ? gtNewBlkIndir(layout, result) : gtNewIndir(lclTyp, result); - } - } - - return result; -} - -/***************************************************************************** - * - * Return true if the given node (excluding children trees) contains side effects. - * Note that it does not recurse, and children need to be handled separately. - * It may return false even if the node has GTF_SIDE_EFFECT (because of its children). - */ - -bool Compiler::gtNodeHasSideEffects(GenTree* tree, GenTreeFlags flags, bool ignoreCctors) -{ - if (flags & GTF_ASG) - { - if (tree->OperRequiresAsgFlag()) - { - return true; - } - } - - // Are there only GTF_CALL side effects remaining? (and no other side effect kinds) - if (flags & GTF_CALL) - { - GenTree* potentialCall = tree; - - if (potentialCall->OperIs(GT_RET_EXPR)) - { - // We need to preserve return expressions where the underlying call - // has side effects. Otherwise early folding can result in us dropping - // the call. - potentialCall = potentialCall->AsRetExpr()->gtInlineCandidate; - } - - if (potentialCall->OperIs(GT_CALL)) - { - GenTreeCall* const call = potentialCall->AsCall(); - const bool ignoreExceptions = (flags & GTF_EXCEPT) == 0; - return call->HasSideEffects(this, ignoreExceptions, ignoreCctors); - } - } - - if (flags & GTF_EXCEPT) - { - if (tree->OperMayThrow(this)) - { - return true; - } - } - - // Expressions declared as CSE by (e.g.) hoisting code are considered to have relevant side - // effects (if we care about GTF_MAKE_CSE). - if ((flags & GTF_MAKE_CSE) && (tree->gtFlags & GTF_MAKE_CSE)) - { - return true; - } - - return false; -} - -/***************************************************************************** - * Returns true if the expr tree has any side effects. - */ - -bool Compiler::gtTreeHasSideEffects(GenTree* tree, GenTreeFlags flags /* = GTF_SIDE_EFFECT*/, bool ignoreCctors) -{ - // These are the side effect flags that we care about for this tree - GenTreeFlags sideEffectFlags = tree->gtFlags & flags; - - // Does this tree have any Side-effect flags set that we care about? - if (sideEffectFlags == 0) - { - // no it doesn't.. - return false; - } - - if ((sideEffectFlags == GTF_CALL) && tree->IsHelperCall()) - { - // Generally all trees that contain GT_CALL nodes are considered to have side-effects. - // However, for some pure helper calls we lie about this. - if (gtNodeHasSideEffects(tree, flags, ignoreCctors)) - { - return true; - } - - // The GTF_CALL may be contributed by an operand, so check for - // that. - bool hasCallInOperand = false; - tree->VisitOperands([=, &hasCallInOperand](GenTree* op) { - if (gtTreeHasSideEffects(op, GTF_CALL, ignoreCctors)) - { - hasCallInOperand = true; - return GenTree::VisitResult::Abort; - } - return GenTree::VisitResult::Continue; - }); - - return hasCallInOperand; - } - - return true; -} - -//------------------------------------------------------------------------ -// gtSplitTree: Split a statement into multiple statements such that a -// specified tree is the first executed non-invariant node in the statement. -// -// Arguments: -// block - The block containing the statement. -// stmt - The statement containing the tree. -// splitPoint - A tree inside the statement. -// firstNewStmt - [out] The first new statement that was introduced. -// [firstNewStmt..stmt) are the statements added by this function. -// splitNodeUse - The use of the tree to split at. -// -// Notes: -// This method turns all non-invariant nodes that would be executed before -// the split point into new separate statements. If those nodes were values -// this involves introducing new locals for those values, such that they can -// be used in the original statement. -// -// This function does not update the flags on the original statement as it is -// expected that the caller is going to modify the use further. Thus, the -// caller is responsible for calling gtUpdateStmtSideEffects on the statement, -// though this is only necessary if the function returned true. -// -// The function may introduce new block ops that need to be morphed when used -// after morph. fgMorphStmtBlockOps can be used on the new statements for -// this purpose. Note that this will invalidate the use, so it should be done -// after any further modifications have been made. -// -// Returns: -// True if any changes were made; false if nothing needed to be done to split the tree. -// -bool Compiler::gtSplitTree( - BasicBlock* block, Statement* stmt, GenTree* splitPoint, Statement** firstNewStmt, GenTree*** splitNodeUse) -{ - class Splitter final : public GenTreeVisitor - { - BasicBlock* m_bb; - Statement* m_splitStmt; - GenTree* m_splitNode; - - struct UseInfo - { - GenTree** Use; - GenTree* User; - }; - ArrayStack m_useStack; - - public: - enum - { - DoPreOrder = true, - DoPostOrder = true, - UseExecutionOrder = true - }; - - Splitter(Compiler* compiler, BasicBlock* bb, Statement* stmt, GenTree* splitNode) - : GenTreeVisitor(compiler) - , m_bb(bb) - , m_splitStmt(stmt) - , m_splitNode(splitNode) - , m_useStack(compiler->getAllocator(CMK_ArrayStack)) - { - } - - Statement* FirstStatement = nullptr; - GenTree** SplitNodeUse = nullptr; - bool MadeChanges = false; - - fgWalkResult PreOrderVisit(GenTree** use, GenTree* user) - { - assert(!(*use)->OperIs(GT_QMARK)); - m_useStack.Push(UseInfo{use, user}); - return WALK_CONTINUE; - } - - fgWalkResult PostOrderVisit(GenTree** use, GenTree* user) - { - if (*use == m_splitNode) - { - bool userIsReturned = false; - // Split all siblings and ancestor siblings. - int i; - for (i = 0; i < m_useStack.Height() - 1; i++) - { - const UseInfo& useInf = m_useStack.BottomRef(i); - if (useInf.Use == use) - { - break; - } - - // If this has the same user as the next node then it is a - // sibling of an ancestor -- and thus not on the "path" - // that contains the split node. - if (m_useStack.BottomRef(i + 1).User == useInf.User) - { - SplitOutUse(useInf, userIsReturned); - } - else - { - // This is an ancestor of the node we're splitting on. - userIsReturned = IsReturned(useInf, userIsReturned); - } - } - - assert(m_useStack.Bottom(i).Use == use); - userIsReturned = IsReturned(m_useStack.BottomRef(i), userIsReturned); - - // The remaining nodes should be operands of the split node. - for (i++; i < m_useStack.Height(); i++) - { - const UseInfo& useInf = m_useStack.BottomRef(i); - assert(useInf.User == *use); - SplitOutUse(useInf, userIsReturned); - } - - SplitNodeUse = use; - - return WALK_ABORT; - } - - while (m_useStack.Top(0).Use != use) - { - m_useStack.Pop(); - } - - return WALK_CONTINUE; - } - - private: - bool IsReturned(const UseInfo& useInf, bool userIsReturned) - { - if (useInf.User != nullptr) - { - if (useInf.User->OperIs(GT_RETURN)) - { - return true; - } - - if (userIsReturned && useInf.User->OperIs(GT_COMMA) && (useInf.Use == &useInf.User->AsOp()->gtOp2)) - { - return true; - } - } - - return false; - } - - bool IsValue(const UseInfo& useInf) - { - GenTree* node = *useInf.Use; - - // Some places create void-typed commas that wrap actual values - // (e.g. VN-based dead store removal), so we need the double check - // here. - if (!node->IsValue()) - { - return false; - } - - node = node->gtEffectiveVal(); - if (!node->IsValue()) - { - return false; - } - - GenTree* user = useInf.User; - - if (user == nullptr) - { - return false; - } - - if (user->OperIs(GT_COMMA) && (&user->AsOp()->gtOp1 == useInf.Use)) - { - return false; - } - - if (user->OperIs(GT_CALL)) - { - for (CallArg& callArg : user->AsCall()->gtArgs.Args()) - { - if ((&callArg.EarlyNodeRef() == useInf.Use) && (callArg.GetLateNode() != nullptr)) - { - return false; - } - } - } - - return true; - } - - void SplitOutUse(const UseInfo& useInf, bool userIsReturned) - { - GenTree** use = useInf.Use; - GenTree* user = useInf.User; - - if ((*use)->IsInvariant()) - { - return; - } - - if ((*use)->OperIs(GT_LCL_VAR) && !m_compiler->lvaGetDesc((*use)->AsLclVarCommon())->IsAddressExposed()) - { - // The splitting we do here should always guarantee that we - // only introduce locals for the tree edges that overlap the - // split point, so it should be ok to avoid creating statements - // for locals that aren't address exposed. Note that this - // relies on it being illegal IR to have a tree edge for a - // register candidate that overlaps with an interfering node. - // - // For example, this optimization would be problematic if IR - // like the following could occur: - // - // CALL - // LCL_VAR V00 - // CALL - // STORE_LCL_VAR(...) (setup) - // LCL_VAR V00 - // - return; - } - -#ifndef TARGET_64BIT - // GT_MUL with GTF_MUL_64RSLT is required to stay with casts on the - // operands. Note that one operand may also be a constant, but we - // would have exited early above for that case. - if ((user != nullptr) && user->OperIs(GT_MUL) && user->Is64RsltMul()) - { - assert((*use)->OperIs(GT_CAST)); - SplitOutUse(UseInfo{&(*use)->AsCast()->gtOp1, *use}, false); - return; - } -#endif - - if ((*use)->OperIs(GT_FIELD_LIST, GT_INIT_VAL)) - { - for (GenTree** operandUse : (*use)->UseEdges()) - { - SplitOutUse(UseInfo{operandUse, *use}, false); - } - return; - } - - Statement* stmt = nullptr; - if (!IsValue(useInf)) - { - GenTree* sideEffects = nullptr; - m_compiler->gtExtractSideEffList(*use, &sideEffects); - if (sideEffects != nullptr) - { - stmt = m_compiler->fgNewStmtFromTree(sideEffects, m_splitStmt->GetDebugInfo()); - } - *use = m_compiler->gtNewNothingNode(); - MadeChanges = true; - } - else - { - unsigned lclNum = m_compiler->lvaGrabTemp(true DEBUGARG("Spilling to split statement for tree")); - - if (varTypeIsStruct(*use) && - ((*use)->IsMultiRegNode() || - (IsReturned(useInf, userIsReturned) && m_compiler->compMethodReturnsMultiRegRetType()))) - { - m_compiler->lvaGetDesc(lclNum)->lvIsMultiRegRet = true; - } - - GenTree* store = m_compiler->gtNewTempStore(lclNum, *use); - stmt = m_compiler->fgNewStmtFromTree(store, m_splitStmt->GetDebugInfo()); - *use = m_compiler->gtNewLclvNode(lclNum, genActualType(*use)); - MadeChanges = true; - } - - if (stmt != nullptr) - { - if (FirstStatement == nullptr) - { - FirstStatement = stmt; - } - - m_compiler->fgInsertStmtBefore(m_bb, m_splitStmt, stmt); - } - } - }; - - Splitter splitter(this, block, stmt, splitPoint); - splitter.WalkTree(stmt->GetRootNodePointer(), nullptr); - *firstNewStmt = splitter.FirstStatement; - *splitNodeUse = splitter.SplitNodeUse; - - return splitter.MadeChanges; -} - -//------------------------------------------------------------------------ -// gtWrapWithSideEffects: Extracts side effects from sideEffectSource (if any) -// and wraps the input tree with a COMMA node with them. -// -// Arguments: -// tree - the expression tree to wrap with side effects (if any) -// it has to be either a side effect free subnode of sideEffectsSource -// or any tree outside sideEffectsSource's hierarchy -// sideEffectsSource - the expression tree to extract side effects from -// sideEffectsFlags - side effect flags to be considered -// ignoreRoot - ignore side effects on the expression root node -// -// Return Value: -// The original tree wrapped with a COMMA node that contains the side effects -// or just the tree itself if sideEffectSource has no side effects. -// -GenTree* Compiler::gtWrapWithSideEffects(GenTree* tree, - GenTree* sideEffectsSource, - GenTreeFlags sideEffectsFlags, - bool ignoreRoot) -{ - GenTree* sideEffects = nullptr; - gtExtractSideEffList(sideEffectsSource, &sideEffects, sideEffectsFlags, ignoreRoot); - if (sideEffects != nullptr) - { - // TODO: assert if tree is a subnode of sideEffectsSource and the tree has its own side effects - // otherwise the resulting COMMA might have some side effects to be duplicated - // It should be possible to be smarter here and allow such cases by extracting the side effects - // properly for this particular case. For now, caller is responsible for avoiding such cases. - - GenTree* comma = gtNewOperNode(GT_COMMA, tree->TypeGet(), sideEffects, tree); - if ((vnStore != nullptr) && tree->gtVNPair.BothDefined() && sideEffectsSource->gtVNPair.BothDefined()) - { - comma->gtVNPair = - vnStore->VNPWithExc(tree->gtVNPair, vnStore->VNPExceptionSet(sideEffectsSource->gtVNPair)); - } - comma->SetMorphed(this); - return comma; - } - return tree; -} - -//------------------------------------------------------------------------ -// gtExtractSideEffList: Extracts side effects from the given expression. -// -// Arguments: -// expr - the expression tree to extract side effects from -// pList - pointer to a (possibly null) node -// flags - side effect flags to be considered -// ignoreRoot - ignore side effects on the expression root node -// -// Notes: -// pList is modified such that the original pList is executed after all side -// effects that were extracted. The original side effect execution order is -// preserved. -// -void Compiler::gtExtractSideEffList(GenTree* expr, - GenTree** pList, - GenTreeFlags flags /* = GTF_SIDE_EFFECT*/, - bool ignoreRoot /* = false */) -{ - class SideEffectExtractor final : public GenTreeVisitor - { - const GenTreeFlags m_flags; - - GenTree* m_result = nullptr; - - public: - enum - { - DoPreOrder = true, - UseExecutionOrder = true - }; - - GenTree* GetResult() - { - return m_result; - } - - SideEffectExtractor(Compiler* compiler, GenTreeFlags flags) - : GenTreeVisitor(compiler) - , m_flags(flags) - { - } - - fgWalkResult PreOrderVisit(GenTree** use, GenTree* user) - { - GenTree* node = *use; - - if (!m_compiler->gtTreeHasSideEffects(node, m_flags)) - { - return Compiler::WALK_SKIP_SUBTREES; - } - - if (m_compiler->gtNodeHasSideEffects(node, m_flags)) - { - if (node->OperIsBlk() && !node->OperIsStoreBlk()) - { - JITDUMP("Replace an unused BLK node [%06d] with a NULLCHECK\n", dspTreeID(node)); - m_compiler->gtChangeOperToNullCheck(node, m_compiler->compCurBB); - } - - Append(node); - return Compiler::WALK_SKIP_SUBTREES; - } - - if (node->OperIs(GT_QMARK)) - { - GenTree* prevSideEffects = m_result; - // Visit children out of order so we know if we can - // completely remove the qmark. We cannot modify the - // condition if we cannot completely remove the qmark, so - // we cannot visit it first. - - GenTreeQmark* qmark = node->AsQmark(); - GenTreeColon* colon = qmark->gtGetOp2()->AsColon(); - - m_result = nullptr; - WalkTree(&colon->gtOp1, colon); - GenTree* thenSideEffects = m_result; - - m_result = nullptr; - WalkTree(&colon->gtOp2, colon); - GenTree* elseSideEffects = m_result; - - m_result = prevSideEffects; - - if ((thenSideEffects == nullptr) && (elseSideEffects == nullptr)) - { - WalkTree(&qmark->gtOp1, qmark); - } - else - { - colon->gtOp1 = (thenSideEffects != nullptr) ? thenSideEffects : m_compiler->gtNewNothingNode(); - colon->gtOp2 = (elseSideEffects != nullptr) ? elseSideEffects : m_compiler->gtNewNothingNode(); - qmark->gtType = TYP_VOID; - colon->gtType = TYP_VOID; - Append(qmark); - } - - return Compiler::WALK_SKIP_SUBTREES; - } - - return Compiler::WALK_CONTINUE; - } - - void Append(GenTree* node) - { - if (m_result == nullptr) - { - m_result = node; - return; - } - - GenTree* comma = m_compiler->gtNewOperNode(GT_COMMA, TYP_VOID, m_result, node); - comma->SetMorphed(m_compiler); - - // Both should have valuenumbers defined for both or for neither - // one (unless we are remorphing, in which case a prior transform - // involving either node may have discarded or otherwise - // invalidated the value numbers). - assert((m_result->gtVNPair.BothDefined() == node->gtVNPair.BothDefined()) || !m_compiler->fgGlobalMorph); - - // Set the ValueNumber 'gtVNPair' for the new GT_COMMA node - // - if ((m_compiler->vnStore != nullptr) && m_result->gtVNPair.BothDefined() && node->gtVNPair.BothDefined()) - { - ValueNumPair op1Exceptions = m_compiler->vnStore->VNPExceptionSet(m_result->gtVNPair); - comma->gtVNPair = m_compiler->vnStore->VNPWithExc(node->gtVNPair, op1Exceptions); - } - - m_result = comma; - } - }; - - SideEffectExtractor extractor(this, flags); - - if (ignoreRoot) - { - for (GenTree* op : expr->Operands()) - { - extractor.WalkTree(&op, nullptr); - } - } - else - { - extractor.WalkTree(&expr, nullptr); - } - - if (*pList != nullptr) - { - extractor.Append(*pList); - } - - *pList = extractor.GetResult(); -} - -/***************************************************************************** - * - * For debugging only - displays a tree node list and makes sure all the - * links are correctly set. - */ - -#ifdef DEBUG -void dispNodeList(GenTree* list, bool verbose) -{ - GenTree* last = nullptr; - GenTree* next; - - if (!list) - { - return; - } - - while (true) - { - next = list->gtNext; - - if (verbose) - { - printf("%08X -> %08X -> %08X\n", last, list, next); - } - - assert(!last || last->gtNext == list); - - assert(next == nullptr || next->gtPrev == list); - - if (!next) - { - break; - } - - last = list; - list = next; - } - printf(""); // null string means flush -} -#endif - -/***************************************************************************** - * Callback to mark the nodes of a qmark-colon subtree that are conditionally - * executed. - */ - -/* static */ -Compiler::fgWalkResult Compiler::gtMarkColonCond(GenTree** pTree, fgWalkData* data) -{ - assert(data->pCallbackData == nullptr); - - (*pTree)->gtFlags |= GTF_COLON_COND; - - return WALK_CONTINUE; -} - -/***************************************************************************** - * Callback to clear the conditionally executed flags of nodes that no longer - will be conditionally executed. Note that when we find another colon we must - stop, as the nodes below this one WILL be conditionally executed. This callback - is called when folding a qmark condition (ie the condition is constant). - */ - -/* static */ -Compiler::fgWalkResult Compiler::gtClearColonCond(GenTree** pTree, fgWalkData* data) -{ - GenTree* tree = *pTree; - - assert(data->pCallbackData == nullptr); - - if (tree->OperGet() == GT_COLON) - { - // Nodes below this will be conditionally executed. - return WALK_SKIP_SUBTREES; - } - - tree->gtFlags &= ~GTF_COLON_COND; - return WALK_CONTINUE; -} - -Compiler::FindLinkData Compiler::gtFindLink(Statement* stmt, GenTree* node) -{ - class FindLinkWalker : public GenTreeVisitor - { - GenTree* m_node; - GenTree** m_edge = nullptr; - GenTree* m_parent = nullptr; - - public: - enum - { - DoPreOrder = true, - }; - - FindLinkWalker(Compiler* comp, GenTree* node) - : GenTreeVisitor(comp) - , m_node(node) - { - } - - FindLinkData GetResult() - { - return FindLinkData{m_node, m_edge, m_parent}; - } - - fgWalkResult PreOrderVisit(GenTree** use, GenTree* user) - { - if (*use == m_node) - { - m_edge = use; - m_parent = user; - return WALK_ABORT; - } - - return WALK_CONTINUE; - } - }; - - FindLinkWalker walker(this, node); - walker.WalkTree(stmt->GetRootNodePointer(), nullptr); - return walker.GetResult(); -} - -/***************************************************************************** - * - * Callback that checks if a tree node has oper type GT_CATCH_ARG - */ - -static Compiler::fgWalkResult gtFindCatchArg(GenTree** pTree, Compiler::fgWalkData* /* data */) -{ - return ((*pTree)->OperGet() == GT_CATCH_ARG) ? Compiler::WALK_ABORT : Compiler::WALK_CONTINUE; -} - -/*****************************************************************************/ -bool Compiler::gtHasCatchArg(GenTree* tree) -{ - if (((tree->gtFlags & GTF_ORDER_SIDEEFF) != 0) && (fgWalkTreePre(&tree, gtFindCatchArg) == WALK_ABORT)) - { - return true; - } - return false; -} - -//------------------------------------------------------------------------ -// gtGetTypeProducerKind: determine if a tree produces a runtime type, and -// if so, how. -// -// Arguments: -// tree - tree to examine -// -// Return Value: -// TypeProducerKind for the tree. -// -// Notes: -// Checks to see if this tree returns a RuntimeType value, and if so, -// how that value is determined. -// -// Currently handles these cases -// 1) The result of Object::GetType -// 2) The result of typeof(...) -// 3) A null reference -// 4) Tree is otherwise known to have type RuntimeType -// -// The null reference case is surprisingly common because operator -// overloading turns the otherwise innocuous -// -// Type t = ....; -// if (t == null) -// -// into a method call. - -Compiler::TypeProducerKind Compiler::gtGetTypeProducerKind(GenTree* tree) -{ - if (tree->gtOper == GT_CALL) - { - if (tree->AsCall()->IsHelperCall()) - { - if (gtIsTypeHandleToRuntimeTypeHelper(tree->AsCall())) - { - return TPK_Handle; - } - } - else if (tree->AsCall()->gtCallMoreFlags & GTF_CALL_M_SPECIAL_INTRINSIC) - { - if (lookupNamedIntrinsic(tree->AsCall()->gtCallMethHnd) == NI_System_Object_GetType) - { - return TPK_GetType; - } - } - } - else if ((tree->gtOper == GT_INTRINSIC) && (tree->AsIntrinsic()->gtIntrinsicName == NI_System_Object_GetType)) - { - return TPK_GetType; - } - else if ((tree->gtOper == GT_CNS_INT) && (tree->AsIntCon()->gtIconVal == 0)) - { - return TPK_Null; - } - else - { - bool isExact = false; - bool isNonNull = false; - CORINFO_CLASS_HANDLE clsHnd = gtGetClassHandle(tree, &isExact, &isNonNull); - - if (clsHnd != NO_CLASS_HANDLE && clsHnd == info.compCompHnd->getBuiltinClass(CLASSID_RUNTIME_TYPE)) - { - return TPK_Other; - } - } - return TPK_Unknown; -} - -//------------------------------------------------------------------------ -// gtIsTypeHandleToRuntimeTypeHelperCall -- see if tree is constructing -// a RuntimeType from a handle -// -// Arguments: -// tree - tree to examine -// -// Return Value: -// True if so - -bool Compiler::gtIsTypeHandleToRuntimeTypeHelper(GenTreeCall* call) -{ - return call->gtCallMethHnd == eeFindHelper(CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPE) || - call->gtCallMethHnd == eeFindHelper(CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPE_MAYBENULL); -} - -//------------------------------------------------------------------------ -// gtIsTypeHandleToRuntimeTypeHandleHelperCall -- see if tree is constructing -// a RuntimeTypeHandle from a handle -// -// Arguments: -// tree - tree to examine -// pHelper - optional pointer to a variable that receives the type of the helper -// -// Return Value: -// True if so -// -bool Compiler::gtIsTypeHandleToRuntimeTypeHandleHelper(GenTreeCall* call, CorInfoHelpFunc* pHelper) -{ - CorInfoHelpFunc helper = CORINFO_HELP_UNDEF; - - if (call->gtCallMethHnd == eeFindHelper(CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPEHANDLE)) - { - helper = CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPEHANDLE; - } - else if (call->gtCallMethHnd == eeFindHelper(CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPEHANDLE_MAYBENULL)) - { - helper = CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPEHANDLE_MAYBENULL; - } - - if (pHelper != nullptr) - { - *pHelper = helper; - } - - return helper != CORINFO_HELP_UNDEF; -} - -//------------------------------------------------------------------------ -// gtTreeContainsOper -- check if the tree contains any subtree with the specified oper. -// -// Arguments: -// tree - tree to examine -// oper - oper to check for -// -// Return Value: -// True if any subtree has the specified oper; otherwise false. -// -bool Compiler::gtTreeContainsOper(GenTree* tree, genTreeOps oper) -{ - class Visitor final : public GenTreeVisitor - { - genTreeOps m_oper; - - public: - Visitor(Compiler* comp, genTreeOps oper) - : GenTreeVisitor(comp) - , m_oper(oper) - { - } - - enum - { - DoPreOrder = true, - }; - - fgWalkResult PreOrderVisit(GenTree** use, GenTree* user) - { - if ((*use)->OperIs(m_oper)) - { - return WALK_ABORT; - } - - return WALK_CONTINUE; - } - }; - - Visitor visitor(this, oper); - return visitor.WalkTree(&tree, nullptr) == WALK_ABORT; -} - -//------------------------------------------------------------------------ -// gtCollectExceptions: walk a tree collecting a bit set of exceptions the tree -// may throw. -// -// Arguments: -// tree - tree to examine -// -// Return Value: -// Bit set of exceptions the tree may throw. -// -ExceptionSetFlags Compiler::gtCollectExceptions(GenTree* tree) -{ - class ExceptionsWalker final : public GenTreeVisitor - { - ExceptionSetFlags m_preciseExceptions = ExceptionSetFlags::None; - - public: - ExceptionsWalker(Compiler* comp) - : GenTreeVisitor(comp) - { - } - - enum - { - DoPreOrder = true, - }; - - ExceptionSetFlags GetFlags() - { - return m_preciseExceptions; - } - - fgWalkResult PreOrderVisit(GenTree** use, GenTree* user) - { - GenTree* tree = *use; - if ((tree->gtFlags & GTF_EXCEPT) == 0) - { - return WALK_SKIP_SUBTREES; - } - - m_preciseExceptions |= tree->OperExceptions(m_compiler); - return WALK_CONTINUE; - } - }; - - // We only expect the caller to ask for precise exceptions for cases where - // it may help with disambiguating between exceptions. If the tree contains - // a call it can always throw arbitrary exceptions. - assert((tree->gtFlags & GTF_CALL) == 0); - - ExceptionsWalker walker(this); - walker.WalkTree(&tree, nullptr); - assert(((tree->gtFlags & GTF_EXCEPT) == 0) || (walker.GetFlags() != ExceptionSetFlags::None)); - return walker.GetFlags(); -} - -//----------------------------------------------------------- -// gtComplexityExceeds: Check if a tree exceeds a specified complexity in terms -// of number of sub nodes. -// -// Arguments: -// tree - The tree to check -// limit - The limit in terms of number of nodes -// complexity - [out, optional] the actual node count (if not greater than limit) -// -// Return Value: -// True if there are more than limit nodes in tree; otherwise false. -// -bool Compiler::gtComplexityExceeds(GenTree* tree, unsigned limit, unsigned* complexity) -{ - struct ComplexityVisitor : GenTreeVisitor - { - enum - { - DoPreOrder = true, - }; - - ComplexityVisitor(Compiler* comp, unsigned limit) - : GenTreeVisitor(comp) - , m_limit(limit) - { - } - - fgWalkResult PreOrderVisit(GenTree** use, GenTree* user) - { - if (++m_numNodes > m_limit) - { - return WALK_ABORT; - } - - return WALK_CONTINUE; - } - - unsigned NumNodes() - { - return m_numNodes; - } - - private: - unsigned m_limit; - unsigned m_numNodes = 0; - }; - - ComplexityVisitor visitor(this, limit); - - fgWalkResult result = visitor.WalkTree(&tree, nullptr); - - if (complexity != nullptr) - { - *complexity = visitor.NumNodes(); - } - - return (result == WALK_ABORT); -} - -bool GenTree::IsPhiNode() -{ - return (OperGet() == GT_PHI_ARG) || (OperGet() == GT_PHI) || IsPhiDefn(); -} - -bool GenTree::IsPhiDefn() -{ - return OperIs(GT_STORE_LCL_VAR) && AsLclVar()->Data()->OperIs(GT_PHI); -} - -bool GenTree::IsLclVarAddr() const -{ - return OperIs(GT_LCL_ADDR) && (AsLclFld()->GetLclOffs() == 0); -} - -// IsPartialLclFld: Check for a GT_LCL_FLD whose type is a different size than the lclVar. -// -// Arguments: -// comp - the Compiler object. -// -// Return Value: -// Returns "true" iff 'this' is a GT_LCL_FLD or GT_STORE_LCL_FLD on which the type -// is not the same size as the type of the GT_LCL_VAR -// -bool GenTree::IsPartialLclFld(Compiler* comp) -{ - return OperIs(GT_LCL_FLD, GT_STORE_LCL_FLD) && - (comp->lvaGetDesc(AsLclFld())->lvExactSize() != AsLclFld()->GetSize()); -} - -//------------------------------------------------------------------------ -// DefinesLocal: Does "this" define a local? -// -// Arguments: -// comp - the compiler instance -// pLclVarTree - [out] parameter for the local representing the definition -// pIsEntire - optional [out] parameter for whether the store represents -// a "full" definition (overwrites the entire variable) -// pOffset - optional [out] parameter for the offset, relative to the -// local, at which the store is performed -// pSize - optional [out] parameter for the amount of bytes affected -// by the store -// -// Return Value: -// Whether "this" represents a store to a possibly tracked local variable -// before rationalization. -// -// Notes: -// This function is contractually bound to recognize a superset of stores -// that "LocalAddressVisitor" recognizes and transforms, as it is used to -// detect which trees can define tracked locals. -// -bool GenTree::DefinesLocal( - Compiler* comp, GenTreeLclVarCommon** pLclVarTree, bool* pIsEntire, ssize_t* pOffset, unsigned* pSize) -{ - assert((pOffset == nullptr) || (*pOffset == 0)); - - if (OperIs(GT_STORE_LCL_VAR)) - { - *pLclVarTree = AsLclVarCommon(); - if (pIsEntire != nullptr) - { - *pIsEntire = true; - } - if (pOffset != nullptr) - { - *pOffset = 0; - } - if (pSize != nullptr) - { - *pSize = comp->lvaLclExactSize(AsLclVarCommon()->GetLclNum()); - } - - return true; - } - if (OperIs(GT_STORE_LCL_FLD)) - { - *pLclVarTree = AsLclVarCommon(); - if (pIsEntire != nullptr) - { - *pIsEntire = !AsLclFld()->IsPartialLclFld(comp); - } - if (pOffset != nullptr) - { - *pOffset = AsLclFld()->GetLclOffs(); - } - if (pSize != nullptr) - { - *pSize = AsLclFld()->GetSize(); - } - - return true; - } - if (OperIs(GT_CALL)) - { - GenTreeLclVarCommon* lclAddr = comp->gtCallGetDefinedRetBufLclAddr(AsCall()); - if (lclAddr == nullptr) - { - return false; - } - - *pLclVarTree = lclAddr; - - if ((pIsEntire != nullptr) || (pSize != nullptr)) - { - unsigned storeSize = comp->typGetObjLayout(AsCall()->gtRetClsHnd)->GetSize(); - - if (pIsEntire != nullptr) - { - *pIsEntire = storeSize == comp->lvaLclExactSize(lclAddr->GetLclNum()); - } - - if (pSize != nullptr) - { - *pSize = storeSize; - } - } - - if (pOffset != nullptr) - { - *pOffset = lclAddr->GetLclOffs(); - } - - return true; - } - - return false; -} - -//------------------------------------------------------------------------ -// IsImplicitByrefParameterValuePreMorph: -// Determine if this tree represents the value of an implicit byref -// parameter, and if so return the tree for the parameter. To be used -// before implicit byrefs have been morphed. -// -// Arguments: -// compiler - compiler instance -// -// Return Value: -// Node for the local, or nullptr. -// -GenTreeLclVarCommon* GenTree::IsImplicitByrefParameterValuePreMorph(Compiler* compiler) -{ -#if FEATURE_IMPLICIT_BYREFS && !defined(TARGET_LOONGARCH64) // TODO-LOONGARCH64-CQ: enable this. - - GenTreeLclVarCommon* lcl = OperIsLocal() ? AsLclVarCommon() : nullptr; - - if ((lcl != nullptr) && compiler->lvaIsImplicitByRefLocal(lcl->GetLclNum())) - { - return lcl; - } - -#endif // FEATURE_IMPLICIT_BYREFS && !defined(TARGET_LOONGARCH64) - - return nullptr; -} - -//------------------------------------------------------------------------ -// IsImplicitByrefParameterValuePostMorph: -// Determine if this tree represents the value of an implicit byref -// parameter, and if so return the tree for the parameter. To be used after -// implicit byrefs have been morphed. -// -// Arguments: -// compiler - compiler instance -// addr - [out] tree representing the address computation on top of the implicit byref. -// Will be the same as the return value if the whole implicit byref is used, for example. -// offset - [out] Offset that "addr" is adding on top of the returned local. -// -// Return Value: -// Node for the local, or nullptr. -// -GenTreeLclVar* GenTree::IsImplicitByrefParameterValuePostMorph(Compiler* compiler, - GenTree** addr, - target_ssize_t* offset) -{ -#if FEATURE_IMPLICIT_BYREFS && !defined(TARGET_LOONGARCH64) // TODO-LOONGARCH64-CQ: enable this. - - if (!OperIsLoad()) - { - return nullptr; - } - - *addr = AsIndir()->Addr(); - - GenTree* innerAddr = *addr; - compiler->gtPeelOffsets(&innerAddr, offset); - - if (innerAddr->OperIs(GT_LCL_VAR)) - { - GenTreeLclVar* lcl = innerAddr->AsLclVar(); - if (compiler->lvaIsImplicitByRefLocal(lcl->GetLclNum())) - { - return lcl; - } - } - -#endif // FEATURE_IMPLICIT_BYREFS && !defined(TARGET_LOONGARCH64) && !defined(TARGET_RISCV64) - - return nullptr; -} - -//------------------------------------------------------------------------ -// IsLclVarUpdateTree: Determine whether this is a local store tree of the -// form Vn = Vn 'oper' 'otherTree' where Vn is a lclVar -// -// Arguments: -// pOtherTree - An "out" argument in which 'otherTree' will be returned. -// pOper - An "out" argument in which 'oper' will be returned. -// -// Return Value: -// If the tree is of the above form, the lclNum of the variable being -// updated is returned, and 'pOtherTree' and 'pOper' are set. -// Otherwise, returns BAD_VAR_NUM. -// -// Notes: -// 'otherTree' can have any shape. -// We avoid worrying about whether the op is commutative by only considering the -// first operand of the rhs. It is expected that most trees of this form will -// already have the lclVar on the lhs. -// TODO-CQ: Evaluate whether there are missed opportunities due to this, or -// whether gtSetEvalOrder will already have put the lclVar on the lhs in -// the cases of interest. - -unsigned GenTree::IsLclVarUpdateTree(GenTree** pOtherTree, genTreeOps* pOper) -{ - if (OperIs(GT_STORE_LCL_VAR) && AsLclVar()->Data()->OperIsBinary()) - { - unsigned lclNum = AsLclVar()->GetLclNum(); - GenTree* value = AsLclVar()->Data(); - GenTree* op1 = value->AsOp()->gtOp1; - GenTree* op2 = value->AsOp()->gtOp2; - - // Some operators, such as LEA, are currently declared as binary but may - // not have two operands. We must check that both operands actually exist. - if ((op1 != nullptr) && (op2 != nullptr) && (op1->OperGet() == GT_LCL_VAR) && - (op1->AsLclVarCommon()->GetLclNum() == lclNum)) - { - *pOtherTree = op2; - *pOper = value->OperGet(); - return lclNum; - } - } - - return BAD_VAR_NUM; -} - -//------------------------------------------------------------------------ -// IsBlockProfileUpdate: Determine whether this tree is updating -// a basic block profile counter -// -// Return Value: -// True if this tree is updating a block profile count -// -bool GenTree::IsBlockProfileUpdate() -{ - return OperIs(GT_STOREIND) && AsIndir()->Addr()->IsIconHandle(GTF_ICON_BBC_PTR); -} - -#ifdef DEBUG -//------------------------------------------------------------------------ -// canBeContained: check whether this tree node may be a subcomponent of its parent for purposes -// of code generation. -// -// Return Value: -// True if it is possible to contain this node and false otherwise. -// -bool GenTree::canBeContained() const -{ - assert(OperIsLIR()); - - if (IsMultiRegLclVar()) - { - return false; - } - - if (gtHasReg(nullptr)) - { - return false; - } - - // It is not possible for nodes that do not produce values or that are not containable values to be contained. - if (!IsValue() || ((DebugOperKind() & DBK_NOCONTAIN) != 0)) - { - return false; - } - else if (OperIsHWIntrinsic() && !isContainableHWIntrinsic()) - { - return isEmbeddedMaskingCompatibleHWIntrinsic(); - } - - return true; -} -#endif // DEBUG - -//------------------------------------------------------------------------ -// isContained: check whether this tree node is a subcomponent of its parent for codegen purposes -// -// Return Value: -// Returns true if there is no code generated explicitly for this node. -// Essentially, it will be rolled into the code generation for the parent. -// -// Assumptions: -// This method relies upon the value of the GTF_CONTAINED flag. -// Therefore this method is only valid after Lowering. -// Also note that register allocation or other subsequent phases may cause -// nodes to become contained (or not) and therefore this property may change. -// -bool GenTree::isContained() const -{ - assert(OperIsLIR()); - const bool isMarkedContained = ((gtFlags & GTF_CONTAINED) != 0); - -#ifdef DEBUG - if (!canBeContained()) - { - assert(!isMarkedContained); - } - - // if it's contained it can't be unused. - if (isMarkedContained) - { - assert(!IsUnusedValue()); - } -#endif // DEBUG - return isMarkedContained; -} - -// return true if node is contained and an indir -bool GenTree::isContainedIndir() const -{ - return OperIsIndir() && isContained(); -} - -bool GenTree::isIndirAddrMode() -{ - return OperIsIndir() && AsIndir()->Addr()->OperIsAddrMode() && AsIndir()->Addr()->isContained(); -} - -bool GenTree::isIndir() const -{ - return OperGet() == GT_IND || OperGet() == GT_STOREIND; -} - -bool GenTreeIndir::HasBase() -{ - return Base() != nullptr; -} - -bool GenTreeIndir::HasIndex() -{ - return Index() != nullptr; -} - -GenTree* GenTreeIndir::Base() -{ - GenTree* addr = Addr(); - - if (isIndirAddrMode()) - { - GenTree* result = addr->AsAddrMode()->Base(); - if (result != nullptr) - { - result = result->gtEffectiveVal(); - } - return result; - } - else - { - return addr; // TODO: why do we return 'addr' here, but we return 'nullptr' in the equivalent Index() case? - } -} - -GenTree* GenTreeIndir::Index() -{ - if (isIndirAddrMode()) - { - GenTree* result = Addr()->AsAddrMode()->Index(); - if (result != nullptr) - { - result = result->gtEffectiveVal(); - } - return result; - } - else - { - return nullptr; - } -} - -unsigned GenTreeIndir::Scale() -{ - if (HasIndex()) - { - return Addr()->AsAddrMode()->gtScale; - } - else - { - return 1; - } -} - -ssize_t GenTreeIndir::Offset() -{ - if (isIndirAddrMode()) - { - return Addr()->AsAddrMode()->Offset(); - } - else if (Addr()->IsCnsIntOrI() && Addr()->isContained()) - { - return Addr()->AsIntConCommon()->IconValue(); - } - else - { - return 0; - } -} - -unsigned GenTreeIndir::Size() const -{ - assert(isIndir() || OperIsBlk()); - return OperIsBlk() ? AsBlk()->Size() : genTypeSize(TypeGet()); -} - -//------------------------------------------------------------------------ -// GenTreeIntConCommon::ImmedValNeedsReloc: does this immediate value needs recording a relocation with the VM? -// -// Arguments: -// comp - Compiler instance -// -// Return Value: -// True if this immediate value requires us to record a relocation for it; false otherwise. - -bool GenTreeIntConCommon::ImmedValNeedsReloc(Compiler* comp) -{ - return comp->opts.compReloc && IsIconHandle(); -} - -//------------------------------------------------------------------------ -// ImmedValCanBeFolded: can this immediate value be folded for op? -// -// Arguments: -// comp - Compiler instance -// op - Tree operator -// -// Return Value: -// True if this immediate value can be folded for op; false otherwise. - -bool GenTreeIntConCommon::ImmedValCanBeFolded(Compiler* comp, genTreeOps op) -{ - // In general, immediate values that need relocations can't be folded. - // There are cases where we do want to allow folding of handle comparisons - // (e.g., typeof(T) == typeof(int)). - return !ImmedValNeedsReloc(comp) || (op == GT_EQ) || (op == GT_NE); -} - -#ifdef TARGET_AMD64 -// Returns true if this absolute address fits within the base of an addr mode. -// On Amd64 this effectively means, whether an absolute indirect address can -// be encoded as 32-bit offset relative to IP or zero. -bool GenTreeIntConCommon::FitsInAddrBase(Compiler* comp) -{ -#ifdef DEBUG - // Early out if PC-rel encoding of absolute addr is disabled. - if (!comp->opts.compEnablePCRelAddr) - { - return false; - } -#endif - - if (comp->opts.compReloc) - { - // During Ngen JIT is always asked to generate relocatable code. - // Hence JIT will try to encode only icon handles as pc-relative offsets. - return IsIconHandle() && (IMAGE_REL_BASED_REL32 == comp->eeGetRelocTypeHint((void*)IconValue())); - } - else - { - // During Jitting, we are allowed to generate non-relocatable code. - // On Amd64 we can encode an absolute indirect addr as an offset relative to zero or RIP. - // An absolute indir addr that can fit within 32-bits can ben encoded as an offset relative - // to zero. All other absolute indir addr could be attempted to be encoded as RIP relative - // based on reloc hint provided by VM. RIP relative encoding is preferred over relative - // to zero, because the former is one byte smaller than the latter. For this reason - // we check for reloc hint first and then whether addr fits in 32-bits next. - // - // VM starts off with an initial state to allow both data and code address to be encoded as - // pc-relative offsets. Hence JIT will attempt to encode all absolute addresses as pc-relative - // offsets. It is possible while jitting a method, an address could not be encoded as a - // pc-relative offset. In that case VM will note the overflow and will trigger re-jitting - // of the method with reloc hints turned off for all future methods. Second time around - // jitting will succeed since JIT will not attempt to encode data addresses as pc-relative - // offsets. Note that JIT will always attempt to relocate code addresses (.e.g call addr). - // After an overflow, VM will assume any relocation recorded is for a code address and will - // emit jump thunk if it cannot be encoded as pc-relative offset. - return (IMAGE_REL_BASED_REL32 == comp->eeGetRelocTypeHint((void*)IconValue())) || FitsInI32(); - } -} - -// Returns true if this icon value is encoded as addr needs recording a relocation with VM -bool GenTreeIntConCommon::AddrNeedsReloc(Compiler* comp) -{ - if (comp->opts.compReloc) - { - // During Ngen JIT is always asked to generate relocatable code. - // Hence JIT will try to encode only icon handles as pc-relative offsets. - return IsIconHandle() && (IMAGE_REL_BASED_REL32 == comp->eeGetRelocTypeHint((void*)IconValue())); - } - else - { - return IMAGE_REL_BASED_REL32 == comp->eeGetRelocTypeHint((void*)IconValue()); - } -} - -#elif defined(TARGET_X86) -// Returns true if this absolute address fits within the base of an addr mode. -// On x86 all addresses are 4-bytes and can be directly encoded in an addr mode. -bool GenTreeIntConCommon::FitsInAddrBase(Compiler* comp) -{ -#ifdef DEBUG - // Early out if PC-rel encoding of absolute addr is disabled. - if (!comp->opts.compEnablePCRelAddr) - { - return false; - } -#endif - - return IsCnsIntOrI(); -} - -// Returns true if this icon value is encoded as addr needs recording a relocation with VM -bool GenTreeIntConCommon::AddrNeedsReloc(Compiler* comp) -{ - // If generating relocatable code, icons should be reported for recording relocatons. - return comp->opts.compReloc && IsIconHandle(); -} -#endif // TARGET_X86 - -#if defined(FEATURE_HW_INTRINSICS) -unsigned GenTreeVecCon::ElementCount(unsigned simdSize, var_types simdBaseType) -{ - return simdSize / genTypeSize(simdBaseType); -} - -bool Compiler::IsValidForShuffle(GenTreeVecCon* vecCon, unsigned simdSize, var_types simdBaseType) const -{ -#if defined(TARGET_XARCH) - size_t elementSize = genTypeSize(simdBaseType); - size_t elementCount = simdSize / elementSize; - - if (simdSize == 32) - { - if (!compOpportunisticallyDependsOn(InstructionSet_AVX2)) - { - // While we could accelerate some functions on hardware with only AVX support - // it's likely not worth it overall given that IsHardwareAccelerated reports false - return false; - } - else if ((varTypeIsByte(simdBaseType) && !compOpportunisticallyDependsOn(InstructionSet_AVX512VBMI_VL)) || - (varTypeIsShort(simdBaseType) && !compOpportunisticallyDependsOn(InstructionSet_AVX512BW_VL))) - { - bool crossLane = false; - - for (size_t index = 0; index < elementCount; index++) - { - uint64_t value = vecCon->GetIntegralVectorConstElement(index, simdBaseType); - - if (value >= elementCount) - { - continue; - } - - if (index < (elementCount / 2)) - { - if (value >= (elementCount / 2)) - { - crossLane = true; - break; - } - } - else if (value < (elementCount / 2)) - { - crossLane = true; - break; - } - } - - if (crossLane) - { - // TODO-XARCH-CQ: We should emulate cross-lane shuffling for byte/sbyte and short/ushort - return false; - } - } - } - else if (simdSize == 64) - { - if (varTypeIsByte(simdBaseType) && !compOpportunisticallyDependsOn(InstructionSet_AVX512VBMI)) - { - // TYP_BYTE, TYP_UBYTE need AVX512VBMI. - return false; - } - } - else - { - assert(simdSize == 16); - - if (varTypeIsSmall(simdBaseType) && !compOpportunisticallyDependsOn(InstructionSet_SSSE3)) - { - // TYP_BYTE, TYP_UBYTE, TYP_SHORT, and TYP_USHORT need SSSE3 to be able to shuffle any operation - return false; - } - } -#endif // TARGET_XARCH - - return true; -} - -//------------------------------------------------------------------------ -// GenTreeVecCon::EvaluateUnaryInPlace: Evaluates this constant using the given operation -// -// Arguments: -// oper - the operation to use in the evaluation -// scalar - true if this is a scalar operation; otherwise, false -// baseType - the base type of the constant being checked -// -void GenTreeVecCon::EvaluateUnaryInPlace(genTreeOps oper, bool scalar, var_types baseType) -{ - switch (gtType) - { - case TYP_SIMD8: - { - simd8_t result = {}; - EvaluateUnarySimd(oper, scalar, baseType, &result, gtSimd8Val); - gtSimd8Val = result; - break; - } - - case TYP_SIMD12: - { - simd12_t result = {}; - EvaluateUnarySimd(oper, scalar, baseType, &result, gtSimd12Val); - gtSimd12Val = result; - break; - } - - case TYP_SIMD16: - { - simd16_t result = {}; - EvaluateUnarySimd(oper, scalar, baseType, &result, gtSimd16Val); - gtSimd16Val = result; - break; - } - -#if defined(TARGET_XARCH) - case TYP_SIMD32: - { - simd32_t result = {}; - EvaluateUnarySimd(oper, scalar, baseType, &result, gtSimd32Val); - gtSimd32Val = result; - break; - } - - case TYP_SIMD64: - { - simd64_t result = {}; - EvaluateUnarySimd(oper, scalar, baseType, &result, gtSimd64Val); - gtSimd64Val = result; - break; - } -#endif // TARGET_XARCH - - default: - { - unreached(); - } - } -} - -//------------------------------------------------------------------------ -// GenTreeVecCon::EvaluateUnaryInPlace: Evaluates this constant using the given operation -// -// Arguments: -// oper - the operation to use in the evaluation -// scalar - true if this is a scalar operation; otherwise, false -// baseType - the base type of the constant being checked -// other - the other vector constant to use in the evaluation -// -void GenTreeVecCon::EvaluateBinaryInPlace(genTreeOps oper, bool scalar, var_types baseType, GenTreeVecCon* other) -{ - switch (gtType) - { - case TYP_SIMD8: - { - simd8_t result = {}; - EvaluateBinarySimd(oper, scalar, baseType, &result, gtSimd8Val, other->gtSimd8Val); - gtSimd8Val = result; - break; - } - - case TYP_SIMD12: - { - simd12_t result = {}; - EvaluateBinarySimd(oper, scalar, baseType, &result, gtSimd12Val, other->gtSimd12Val); - gtSimd12Val = result; - break; - } - - case TYP_SIMD16: - { - simd16_t result = {}; - EvaluateBinarySimd(oper, scalar, baseType, &result, gtSimd16Val, other->gtSimd16Val); - gtSimd16Val = result; - break; - } - -#if defined(TARGET_XARCH) - case TYP_SIMD32: - { - simd32_t result = {}; - EvaluateBinarySimd(oper, scalar, baseType, &result, gtSimd32Val, other->gtSimd32Val); - gtSimd32Val = result; - break; - } - - case TYP_SIMD64: - { - simd64_t result = {}; - EvaluateBinarySimd(oper, scalar, baseType, &result, gtSimd64Val, other->gtSimd64Val); - gtSimd64Val = result; - break; - } -#endif // TARGET_XARCH - - default: - { - unreached(); - } - } -} - -//------------------------------------------------------------------------ -// GenTreeVecCon::EvaluateBroadcastInPlace: Evaluates this constant using a broadcast -// -// Arguments: -// baseType - the base type of the constant being checked -// scalar - the value to broadcast as part of the evaluation -// -void GenTreeVecCon::EvaluateBroadcastInPlace(var_types baseType, double scalar) -{ - switch (baseType) - { - case TYP_FLOAT: - { - EvaluateBroadcastInPlace(static_cast(scalar)); - break; - } - - case TYP_DOUBLE: - { - EvaluateBroadcastInPlace(static_cast(scalar)); - break; - } - - default: - { - unreached(); - } - } -} - -//------------------------------------------------------------------------ -// GenTreeVecCon::EvaluateBroadcastInPlace: Evaluates this constant using a broadcast -// -// Arguments: -// baseType - the base type of the constant being checked -// scalar - the value to broadcast as part of the evaluation -// -void GenTreeVecCon::EvaluateBroadcastInPlace(var_types baseType, int64_t scalar) -{ - switch (baseType) - { - case TYP_BYTE: - { - EvaluateBroadcastInPlace(static_cast(scalar)); - break; - } - - case TYP_UBYTE: - { - EvaluateBroadcastInPlace(static_cast(scalar)); - break; - } - - case TYP_SHORT: - { - EvaluateBroadcastInPlace(static_cast(scalar)); - break; - } - - case TYP_USHORT: - { - EvaluateBroadcastInPlace(static_cast(scalar)); - break; - } - - case TYP_INT: - { - EvaluateBroadcastInPlace(static_cast(scalar)); - break; - } - - case TYP_UINT: - { - EvaluateBroadcastInPlace(static_cast(scalar)); - break; - } - - case TYP_LONG: - { - EvaluateBroadcastInPlace(static_cast(scalar)); - break; - } - - case TYP_ULONG: - { - EvaluateBroadcastInPlace(static_cast(scalar)); - break; - } - - default: - { - unreached(); - } - } -} - -//------------------------------------------------------------------------ -// GenTreeVecCon::IsBroadcast: Determines if this vector constant is a broadcast -// -// Arguments: -// simdBaseType - the base type of the constant being checked -// -// Returns: -// true if the constant represents a broadcast value; otherwise, false -// -bool GenTreeVecCon::IsBroadcast(var_types simdBaseType) const -{ - assert(varTypeIsSIMD(gtType)); - assert(varTypeIsArithmetic(simdBaseType)); - - int elementCount = ElementCount(genTypeSize(gtType), simdBaseType); - - switch (simdBaseType) - { - case TYP_BYTE: - case TYP_UBYTE: - { - return ElementsAreSame(>SimdVal.u8[0], elementCount); - } - - case TYP_SHORT: - case TYP_USHORT: - { - return ElementsAreSame(>SimdVal.u16[0], elementCount); - } - - case TYP_FLOAT: - case TYP_INT: - case TYP_UINT: - { - return ElementsAreSame(>SimdVal.u32[0], elementCount); - } - - case TYP_DOUBLE: - case TYP_LONG: - case TYP_ULONG: - { - return ElementsAreSame(>SimdVal.u64[0], elementCount); - } - - default: - { - return false; - } - } -} - -//------------------------------------------------------------------------ -// GenTreeVecCon::IsNaN: Determines if this vector constant has all elements being NaN -// -// Arguments: -// simdBaseType - the base type of the constant being checked -// -// Returns: -// true if the constant has all elements being NaN; otherwise, false -// -bool GenTreeVecCon::IsNaN(var_types simdBaseType) const -{ - assert(varTypeIsFloating(simdBaseType)); - uint32_t elementCount = ElementCount(genTypeSize(gtType), simdBaseType); - - for (uint32_t i = 0; i < elementCount; i++) - { - double element = GetElementFloating(simdBaseType, i); - - if (!FloatingPointUtils::isNaN(element)) - { - return false; - } - } - - return true; -} - -//------------------------------------------------------------------------ -// GenTreeVecCon::IsNaN: Determines if this vector constant has all elements being -0 -// -// Arguments: -// simdBaseType - the base type of the constant being checked -// -// Returns: -// true if the constant has all elements being -0; otherwise, false -// -bool GenTreeVecCon::IsNegativeZero(var_types simdBaseType) const -{ - assert(varTypeIsFloating(simdBaseType)); - uint32_t elementCount = ElementCount(genTypeSize(gtType), simdBaseType); - - for (uint32_t i = 0; i < elementCount; i++) - { - double element = GetElementFloating(simdBaseType, i); - - if (!FloatingPointUtils::isNegativeZero(element)) - { - return false; - } - } - - return true; -} - -//------------------------------------------------------------------------ -// GenTreeMskCon::EvaluateUnaryInPlace: Evaluates this constant using the given operation -// -// Arguments: -// oper - the operation to use in the evaluation -// scalar - true if this is a scalar operation; otherwise, false -// baseType - the base type of the constant being checked -// simdSize - the size of the SIMD node the mask is for -// -void GenTreeMskCon::EvaluateUnaryInPlace(genTreeOps oper, bool scalar, var_types baseType, unsigned simdSize) -{ -#if defined(FEATURE_MASKED_HW_INTRINSICS) - simdmask_t result = {}; - EvaluateUnaryMask(oper, scalar, baseType, simdSize, &result, gtSimdMaskVal); - gtSimdMaskVal = result; -#else - unreached(); -#endif // FEATURE_MASKED_HW_INTRINSICS -} - -//------------------------------------------------------------------------ -// GenTreeMskCon::EvaluateUnaryInPlace: Evaluates this constant using the given operation -// -// Arguments: -// oper - the operation to use in the evaluation -// scalar - true if this is a scalar operation; otherwise, false -// baseType - the base type of the constant being checked -// other - the other vector constant to use in the evaluation -// simdSize - the size of the SIMD node the mask is for -// -void GenTreeMskCon::EvaluateBinaryInPlace( - genTreeOps oper, bool scalar, var_types baseType, unsigned simdSize, GenTreeMskCon* other) -{ -#if defined(FEATURE_MASKED_HW_INTRINSICS) - simdmask_t result = {}; - EvaluateBinaryMask(oper, scalar, baseType, simdSize, &result, gtSimdMaskVal, other->gtSimdMaskVal); - gtSimdMaskVal = result; -#else - unreached(); -#endif // FEATURE_MASKED_HW_INTRINSICS -} -#endif // FEATURE_HW_INTRINSICS*/ - -//------------------------------------------------------------------------ -// IsFieldAddr: Is "this" a static or class field address? -// -// Recognizes the following patterns: -// this: ADD(baseAddr, CONST [FldSeq]) -// this: ADD(CONST [FldSeq], baseAddr) -// this: CONST [FldSeq] -// -// Arguments: -// comp - the Compiler object -// pBaseAddr - [out] parameter for "the base address" -// pFldSeq - [out] parameter for the field sequence -// pOffset - [out] parameter for the offset of the component struct fields -// -// Return Value: -// If "this" matches patterns denoted above, with a valid FldSeq, this method -// will return "true" and set "pBaseAddr" to some value, which must be used by -// the caller as the key into the "first field map" to obtain the value for the -// field, or to "nullptr", in which case the handle for the field is the primary -// selector. For instance fields, "base address" will be the object reference, -// for statics - the address to which the field offset with the field sequence -// is added, see "impImportStaticFieldAccess" and "fgMorphFieldAddr". -// -bool GenTree::IsFieldAddr(Compiler* comp, GenTree** pBaseAddr, FieldSeq** pFldSeq, ssize_t* pOffset) -{ - assert(TypeIs(TYP_I_IMPL, TYP_BYREF, TYP_REF)); - - *pBaseAddr = nullptr; - *pFldSeq = nullptr; - - GenTree* baseAddr = nullptr; - FieldSeq* fldSeq = nullptr; - ssize_t offset = 0; - - if (OperIs(GT_ADD)) - { - if (AsOp()->gtOp2->IsCnsIntOrI()) - { - baseAddr = AsOp()->gtOp1; - fldSeq = AsOp()->gtOp2->AsIntCon()->gtFieldSeq; - offset = AsOp()->gtOp2->AsIntCon()->IconValue(); - - if ((fldSeq != nullptr) && (fldSeq->GetKind() == FieldSeq::FieldKind::SimpleStaticKnownAddress)) - { - // fldSeq represents a known address (not a small offset) - bail out. - return false; - } - } - else - { - return false; - } - } - else if (IsIconHandle(GTF_ICON_STATIC_HDL)) - { - fldSeq = AsIntCon()->gtFieldSeq; - offset = AsIntCon()->IconValue(); - assert((fldSeq == nullptr) || (fldSeq->GetKind() == FieldSeq::FieldKind::SimpleStaticKnownAddress)); - } - else - { - return false; - } - - if (fldSeq == nullptr) - { - return false; - } - - // Subtract from the offset such that the portion remaining is relative to the field itself. - offset -= fldSeq->GetOffset(); - - // The above screens out obviously invalid cases, but we have more checks to perform. The - // sequence returned from this method *must* start with either a class (NOT struct) field - // or a static field. To avoid the expense of calling "getFieldClass" here, we will instead - // rely on the invariant that TYP_REF base addresses can never appear for struct fields - we - // will effectively treat such cases ("possible" in unsafe code) as undefined behavior. - if (fldSeq->IsStaticField()) - { - // For shared statics, we must encode the logical instantiation argument. - if (fldSeq->IsSharedStaticField()) - { - *pBaseAddr = baseAddr; - } - - *pFldSeq = fldSeq; - *pOffset = offset; - return true; - } - - if (baseAddr->TypeIs(TYP_REF)) - { - assert(!comp->eeIsValueClass(comp->info.compCompHnd->getFieldClass(fldSeq->GetFieldHandle()))); - - *pBaseAddr = baseAddr; - *pFldSeq = fldSeq; - *pOffset = offset; - return true; - } - - // This case is reached, for example, if we have a chain of struct fields that are based on - // some pointer. We do not model such cases because we do not model maps for ByrefExposed - // memory, as it does not have the non-aliasing property of GcHeap and reference types. - return false; -} - -//------------------------------------------------------------------------ -// gtStoreDefinesField: Does the given parent store modify the given field? -// -// Arguments: -// fieldVarDsc - The field local -// offset - Offset of the store, relative to the parent -// size - Size of the store in bytes -// pFieldStoreOffset - [out] parameter for the store's offset relative -// to the field local itself -// pFieldStoreSize - [out] parameter for the amount of the field's -// local's bytes affected by the store -// -// Return Value: -// If the given store affects the given field local, "true, "false" -// otherwise. -// -bool Compiler::gtStoreDefinesField( - LclVarDsc* fieldVarDsc, ssize_t offset, unsigned size, ssize_t* pFieldStoreOffset, unsigned* pFieldStoreSize) -{ - ssize_t fieldOffset = fieldVarDsc->lvFldOffset; - unsigned fieldSize = genTypeSize(fieldVarDsc); // No TYP_STRUCT field locals. - - ssize_t storeEndOffset = offset + static_cast(size); - ssize_t fieldEndOffset = fieldOffset + static_cast(fieldSize); - if ((fieldOffset < storeEndOffset) && (offset < fieldEndOffset)) - { - *pFieldStoreOffset = (offset < fieldOffset) ? 0 : (offset - fieldOffset); - *pFieldStoreSize = static_cast(min(storeEndOffset, fieldEndOffset) - max(offset, fieldOffset)); - - return true; - } - - return false; -} - -//------------------------------------------------------------------------ -// gtGetClassHandle: find class handle for a ref type -// -// Arguments: -// tree -- tree to find handle for -// pIsExact [out] -- whether handle is exact type -// pIsNonNull [out] -- whether tree value is known not to be null -// -// Return Value: -// nullptr if class handle is unknown, -// otherwise the class handle. -// *pIsExact set true if tree type is known to be exactly the handle type, -// otherwise actual type may be a subtype. -// *pIsNonNull set true if tree value is known not to be null, -// otherwise a null value is possible. -// -CORINFO_CLASS_HANDLE Compiler::gtGetClassHandle(GenTree* tree, bool* pIsExact, bool* pIsNonNull) -{ - // Set default values for our out params. - *pIsNonNull = false; - *pIsExact = false; - - // Bail out if the tree is not a ref type. - if (!tree->TypeIs(TYP_REF)) - { - return NO_CLASS_HANDLE; - } - - // Tunnel through commas. - GenTree* obj = tree->gtEffectiveVal(); - const genTreeOps objOp = obj->OperGet(); - CORINFO_CLASS_HANDLE objClass = NO_CLASS_HANDLE; - - switch (objOp) - { - case GT_COMMA: - { - // gtEffectiveVal above means we shouldn't see commas here. - assert(!"unexpected GT_COMMA"); - break; - } - - case GT_LCL_VAR: - { - // For locals, pick up type info from the local table. - const unsigned objLcl = obj->AsLclVar()->GetLclNum(); - - objClass = lvaTable[objLcl].lvClassHnd; - *pIsExact = lvaTable[objLcl].lvClassIsExact; - break; - } - - case GT_CNS_INT: - { - if (obj->IsIconHandle(GTF_ICON_OBJ_HDL)) - { - objClass = info.compCompHnd->getObjectType((CORINFO_OBJECT_HANDLE)obj->AsIntCon()->IconValue()); - if (objClass != NO_CLASS_HANDLE) - { - // if we managed to get a class handle it's definitely not null - *pIsNonNull = true; - *pIsExact = true; - } - } - break; - } - - case GT_RET_EXPR: - { - // If we see a RET_EXPR, recurse through to examine the - // return value expression. - GenTree* retExpr = obj->AsRetExpr()->gtInlineCandidate; - objClass = gtGetClassHandle(retExpr, pIsExact, pIsNonNull); - break; - } - - case GT_CALL: - { - GenTreeCall* call = obj->AsCall(); - if (call->gtCallMoreFlags & GTF_CALL_M_SPECIAL_INTRINSIC) - { - NamedIntrinsic ni = lookupNamedIntrinsic(call->gtCallMethHnd); - if ((ni == NI_System_Array_Clone) || (ni == NI_System_Object_MemberwiseClone)) - { - objClass = gtGetClassHandle(call->gtArgs.GetThisArg()->GetNode(), pIsExact, pIsNonNull); - break; - } - - CORINFO_CLASS_HANDLE specialObjClass = impGetSpecialIntrinsicExactReturnType(call); - if (specialObjClass != nullptr) - { - objClass = specialObjClass; - *pIsExact = true; - *pIsNonNull = true; - break; - } - } - if (call->IsInlineCandidate() && !call->IsGuardedDevirtualizationCandidate()) - { - // For inline candidates, we've already cached the return - // type class handle in the inline info (for GDV candidates, - // this data is valid only for a correct guess, so we cannot - // use it). - InlineCandidateInfo* inlInfo = call->GetSingleInlineCandidateInfo(); - assert(inlInfo != nullptr); - - // Grab it as our first cut at a return type. - assert(inlInfo->methInfo.args.retType == CORINFO_TYPE_CLASS); - objClass = inlInfo->methInfo.args.retTypeClass; - - // If the method is shared, the above may not capture - // the most precise return type information (that is, - // it may represent a shared return type and as such, - // have instances of __Canon). See if we can use the - // context to get at something more definite. - // - // For now, we do this here on demand rather than when - // processing the call, but we could/should apply - // similar sharpening to the argument and local types - // of the inlinee. - if (eeIsSharedInst(objClass)) - { - CORINFO_CONTEXT_HANDLE context = inlInfo->exactContextHandle; - - if (context != nullptr) - { - CORINFO_CLASS_HANDLE exactClass = eeGetClassFromContext(context); - - // Grab the signature in this context. - CORINFO_SIG_INFO sig; - eeGetMethodSig(call->gtCallMethHnd, &sig, exactClass); - assert(sig.retType == CORINFO_TYPE_CLASS); - objClass = sig.retTypeClass; - } - } - } - else if (call->gtCallType == CT_USER_FUNC) - { - // For user calls, we can fetch the approximate return - // type info from the method handle. Unfortunately - // we've lost the exact context, so this is the best - // we can do for now. - CORINFO_METHOD_HANDLE method = call->gtCallMethHnd; - CORINFO_CLASS_HANDLE exactClass = nullptr; - CORINFO_SIG_INFO sig; - eeGetMethodSig(method, &sig, exactClass); - if (sig.retType == CORINFO_TYPE_VOID) - { - // This is a constructor call. - const unsigned methodFlags = info.compCompHnd->getMethodAttribs(method); - assert((methodFlags & CORINFO_FLG_CONSTRUCTOR) != 0); - objClass = info.compCompHnd->getMethodClass(method); - *pIsExact = true; - *pIsNonNull = true; - } - else - { - assert(sig.retType == CORINFO_TYPE_CLASS); - objClass = sig.retTypeClass; - } - } - else if (call->IsHelperCall()) - { - objClass = gtGetHelperCallClassHandle(call, pIsExact, pIsNonNull); - } - - break; - } - - case GT_INTRINSIC: - { - GenTreeIntrinsic* intrinsic = obj->AsIntrinsic(); - - if (intrinsic->gtIntrinsicName == NI_System_Object_GetType) - { - CORINFO_CLASS_HANDLE runtimeType = info.compCompHnd->getBuiltinClass(CLASSID_RUNTIME_TYPE); - assert(runtimeType != NO_CLASS_HANDLE); - - objClass = runtimeType; - *pIsNonNull = true; - } - - break; - } - - case GT_CNS_STR: - { - // For literal strings, we know the class and that the - // value is not null. - objClass = impGetStringClass(); - *pIsExact = true; - *pIsNonNull = true; - break; - } - - case GT_IND: - { - GenTreeIndir* indir = obj->AsIndir(); - GenTree* base = indir->Base(); - - // indir(lcl_var_addr) --> lcl - // - // This comes up during constrained callvirt on ref types. - // - if (base->IsLclVarAddr()) - { - const unsigned objLcl = base->AsLclVarCommon()->GetLclNum(); - objClass = lvaTable[objLcl].lvClassHnd; - *pIsExact = lvaTable[objLcl].lvClassIsExact; - } - else if (base->OperIs(GT_INDEX_ADDR, GT_ARR_ELEM)) - { - // indir(arr_elem(...)) -> array element type - - if (base->OperIs(GT_INDEX_ADDR)) - { - objClass = gtGetArrayElementClassHandle(base->AsIndexAddr()->Arr()); - } - else - { - objClass = gtGetArrayElementClassHandle(base->AsArrElem()->gtArrObj); - } - } - else if (base->OperGet() == GT_ADD) - { - // TODO-VNTypes: use "IsFieldAddr" here instead. - - // This could be a static field access. - // - // See if op1 is a static field base helper call - // and if so, op2 will have the field info. - GenTree* op1 = base->AsOp()->gtOp1; - GenTree* op2 = base->AsOp()->gtOp2; - - if (op2->IsCnsIntOrI()) - { - FieldSeq* fieldSeq = op2->AsIntCon()->gtFieldSeq; - if ((fieldSeq != nullptr) && (fieldSeq->GetOffset() == op2->AsIntCon()->IconValue())) - { - // No benefit to calling gtGetFieldClassHandle here, as - // the exact field being accessed can vary. - CORINFO_FIELD_HANDLE fieldHnd = fieldSeq->GetFieldHandle(); - CORINFO_CLASS_HANDLE fieldOwner = NO_CLASS_HANDLE; - CORINFO_CLASS_HANDLE fieldClass = NO_CLASS_HANDLE; - - // fieldOwner helps us to get a more exact field class for instance fields - if (!fieldSeq->IsStaticField()) - { - bool objIsExact, objIsNonNull; - fieldOwner = gtGetClassHandle(op1, &objIsExact, &objIsNonNull); - } - - if (eeGetFieldType(fieldHnd, &fieldClass, fieldOwner) == TYP_REF) - { - objClass = fieldClass; - } - } - } - } - else if (base->IsIconHandle(GTF_ICON_CONST_PTR, GTF_ICON_STATIC_HDL)) - { - // Check if we have IND(ICON_HANDLE) that represents a static field - FieldSeq* fldSeq = base->AsIntCon()->gtFieldSeq; - if ((fldSeq != nullptr) && (fldSeq->GetOffset() == base->AsIntCon()->IconValue())) - { - CORINFO_FIELD_HANDLE fldHandle = fldSeq->GetFieldHandle(); - objClass = gtGetFieldClassHandle(fldHandle, pIsExact, pIsNonNull); - } - } - else if (base->OperIs(GT_FIELD_ADDR)) - { - objClass = gtGetFieldClassHandle(base->AsFieldAddr()->gtFldHnd, pIsExact, pIsNonNull); - } - break; - } - - case GT_BOX: - { - // Box should just wrap a local var reference which has - // the type we're looking for. Also box only represents a - // non-nullable value type so result cannot be null. - GenTreeBox* box = obj->AsBox(); - GenTree* boxTemp = box->BoxOp(); - assert(boxTemp->IsLocal()); - const unsigned boxTempLcl = boxTemp->AsLclVar()->GetLclNum(); - objClass = lvaTable[boxTempLcl].lvClassHnd; - *pIsExact = lvaTable[boxTempLcl].lvClassIsExact; - *pIsNonNull = true; - break; - } - - default: - { - break; - } - } - - if ((objClass == NO_CLASS_HANDLE) && (vnStore != nullptr)) - { - // Try VN if we haven't found a class handle yet - objClass = vnStore->GetObjectType(tree->gtVNPair.GetConservative(), pIsExact, pIsNonNull); - } - - if ((objClass != NO_CLASS_HANDLE) && !*pIsExact && JitConfig.JitEnableExactDevirtualization()) - { - CORINFO_CLASS_HANDLE exactClass; - if (info.compCompHnd->getExactClasses(objClass, 1, &exactClass) == 1) - { - *pIsExact = true; - objClass = exactClass; - } - else - { - *pIsExact = info.compCompHnd->isExactType(objClass); - } - } - - return objClass; -} - -//------------------------------------------------------------------------ -// gtGetHelperCallClassHandle: find class handle for return value of a -// helper call -// -// Arguments: -// call - helper call to examine -// pIsExact - [OUT] true if type is known exactly -// pIsNonNull - [OUT] true if return value is not null -// -// Return Value: -// nullptr if helper call result is not a ref class, or the class handle -// is unknown, otherwise the class handle. -// -CORINFO_CLASS_HANDLE Compiler::gtGetHelperCallClassHandle(GenTreeCall* call, bool* pIsExact, bool* pIsNonNull) -{ - assert(call->IsHelperCall()); - - *pIsNonNull = false; - *pIsExact = false; - CORINFO_CLASS_HANDLE objClass = nullptr; - const CorInfoHelpFunc helper = eeGetHelperNum(call->gtCallMethHnd); - - switch (helper) - { - case CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPE: - case CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPE_MAYBENULL: - { - // Note for some runtimes these helpers return exact types. - // - // But in those cases the types are also sealed, so there's no - // need to claim exactness here. - const bool helperResultNonNull = (helper == CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPE); - CORINFO_CLASS_HANDLE runtimeType = info.compCompHnd->getBuiltinClass(CLASSID_RUNTIME_TYPE); - - assert(runtimeType != NO_CLASS_HANDLE); - - objClass = runtimeType; - *pIsNonNull = helperResultNonNull; - break; - } - - case CORINFO_HELP_BOX: - case CORINFO_HELP_BOX_NULLABLE: - { - GenTree* typeArg = call->gtArgs.GetUserArgByIndex(0)->GetNode(); - if (typeArg->IsIconHandle(GTF_ICON_CLASS_HDL)) - { - const bool isNullableHelper = (helper == CORINFO_HELP_BOX_NULLABLE); - - objClass = gtGetHelperArgClassHandle(typeArg); - if ((objClass != NO_CLASS_HANDLE) && isNullableHelper) - { - // Nullable is boxed as just T (via CORINFO_HELP_BOX_NULLABLE) - objClass = info.compCompHnd->getTypeForBox(objClass); - } - - if (objClass != NO_CLASS_HANDLE) - { - // CORINFO_HELP_BOX_NULLABLE may return null - // CORINFO_HELP_BOX always returns non-null - *pIsNonNull = !isNullableHelper; - *pIsExact = true; - } - } - } - break; - - case CORINFO_HELP_CHKCASTCLASS: - case CORINFO_HELP_CHKCASTANY: - case CORINFO_HELP_CHKCASTARRAY: - case CORINFO_HELP_CHKCASTINTERFACE: - case CORINFO_HELP_CHKCASTCLASS_SPECIAL: - case CORINFO_HELP_ISINSTANCEOFINTERFACE: - case CORINFO_HELP_ISINSTANCEOFARRAY: - case CORINFO_HELP_ISINSTANCEOFCLASS: - case CORINFO_HELP_ISINSTANCEOFANY: - { - // Fetch the class handle from the helper call arglist - GenTree* typeArg = call->gtArgs.GetArgByIndex(0)->GetNode(); - CORINFO_CLASS_HANDLE castHnd = gtGetHelperArgClassHandle(typeArg); - - // We generally assume the type being cast to is the best type - // for the result, unless it is an interface type. - // - // TODO-CQ: when we have default interface methods then - // this might not be the best assumption. A similar issue arises when - // typing the temp in impCastClassOrIsInstToTree, when we - // expand the cast inline. - if (castHnd != nullptr) - { - DWORD attrs = info.compCompHnd->getClassAttribs(castHnd); - - if ((attrs & CORINFO_FLG_INTERFACE) != 0) - { - castHnd = nullptr; - } - } - - // If we don't have a good estimate for the type we can use the - // type from the value being cast instead. - if (castHnd == nullptr) - { - GenTree* valueArg = call->gtArgs.GetArgByIndex(1)->GetNode(); - castHnd = gtGetClassHandle(valueArg, pIsExact, pIsNonNull); - } - - // We don't know at jit time if the cast will succeed or fail, but if it - // fails at runtime then an exception is thrown for cast helpers, or the - // result is set null for instance helpers. - // - // So it safe to claim the result has the cast type. - // Note we don't know for sure that it is exactly this type. - if (castHnd != nullptr) - { - objClass = castHnd; - } - - break; - } - - case CORINFO_HELP_NEWARR_1_DIRECT: - case CORINFO_HELP_NEWARR_1_MAYBEFROZEN: - case CORINFO_HELP_NEWARR_1_OBJ: - case CORINFO_HELP_NEWARR_1_VC: - case CORINFO_HELP_NEWARR_1_ALIGN8: - case CORINFO_HELP_READYTORUN_NEWARR_1: - { - CORINFO_CLASS_HANDLE arrayHnd = (CORINFO_CLASS_HANDLE)call->compileTimeHelperArgumentHandle; - - if (arrayHnd != NO_CLASS_HANDLE) - { - objClass = arrayHnd; - *pIsExact = true; - *pIsNonNull = true; - } - break; - } - - default: - break; - } - - return objClass; -} - -//------------------------------------------------------------------------ -// gtGetArrayElementClassHandle: find class handle for elements of an array -// of ref types -// -// Arguments: -// array -- array to find handle for -// -// Return Value: -// nullptr if element class handle is unknown, otherwise the class handle. -// -CORINFO_CLASS_HANDLE Compiler::gtGetArrayElementClassHandle(GenTree* array) -{ - bool isArrayExact = false; - bool isArrayNonNull = false; - CORINFO_CLASS_HANDLE arrayClassHnd = gtGetClassHandle(array, &isArrayExact, &isArrayNonNull); - - if (arrayClassHnd != nullptr) - { - // We know the class of the reference - DWORD attribs = info.compCompHnd->getClassAttribs(arrayClassHnd); - - if ((attribs & CORINFO_FLG_ARRAY) != 0) - { - // We know for sure it is an array - CORINFO_CLASS_HANDLE elemClassHnd = nullptr; - CorInfoType arrayElemType = info.compCompHnd->getChildType(arrayClassHnd, &elemClassHnd); - - if (arrayElemType == CORINFO_TYPE_CLASS) - { - // We know it is an array of ref types - return elemClassHnd; - } - } - } - - return nullptr; -} - -//------------------------------------------------------------------------ -// gtGetFieldClassHandle: find class handle for a field -// -// Arguments: -// fieldHnd - field handle for field in question -// pIsExact - [OUT] true if type is known exactly -// pIsNonNull - [OUT] true if field value is not null -// -// Return Value: -// nullptr if helper call result is not a ref class, or the class handle -// is unknown, otherwise the class handle. -// -// May examine runtime state of static field instances. -// -CORINFO_CLASS_HANDLE Compiler::gtGetFieldClassHandle(CORINFO_FIELD_HANDLE fieldHnd, bool* pIsExact, bool* pIsNonNull) -{ - *pIsExact = false; - *pIsNonNull = false; - - CORINFO_CLASS_HANDLE fieldClass = NO_CLASS_HANDLE; - CorInfoType fieldCorType = info.compCompHnd->getFieldType(fieldHnd, &fieldClass); - - if (fieldCorType == CORINFO_TYPE_CLASS) - { - // Optionally, look at the actual type of the field's value - bool queryForCurrentClass = true; - INDEBUG(queryForCurrentClass = (JitConfig.JitQueryCurrentStaticFieldClass() > 0);); - - if (queryForCurrentClass) - { -#if DEBUG - if (verbose || JitConfig.EnableExtraSuperPmiQueries()) - { - char fieldNameBuffer[128]; - char classNameBuffer[128]; - const char* fieldName = eeGetFieldName(fieldHnd, true, fieldNameBuffer, sizeof(fieldNameBuffer)); - const char* className = eeGetClassName(fieldClass, classNameBuffer, sizeof(classNameBuffer)); - JITDUMP("\nQuerying runtime about current class of field %s (declared as %s)\n", fieldName, className); - } -#endif // DEBUG - - // Is this a fully initialized init-only static field? - // - // Note we're not asking for speculative results here, yet. - CORINFO_CLASS_HANDLE currentClass = info.compCompHnd->getStaticFieldCurrentClass(fieldHnd); - - if (currentClass != NO_CLASS_HANDLE) - { - // Yes! We know the class exactly and can rely on this to always be true. - fieldClass = currentClass; - *pIsExact = true; - *pIsNonNull = true; -#ifdef DEBUG - if (verbose || JitConfig.EnableExtraSuperPmiQueries()) - { - char classNameBuffer2[128]; - const char* className2 = eeGetClassName(fieldClass, classNameBuffer2, sizeof(classNameBuffer2)); - JITDUMP("Runtime reports field is init-only and initialized and has class %s\n", className2); - } -#endif // DEBUG - } - else - { - JITDUMP("Field's current class not available\n"); - } - - return fieldClass; - } - } - - return NO_CLASS_HANDLE; -} - -//------------------------------------------------------------------------ -// gtIsTypeof: Checks if the tree is a typeof() -// -// Arguments: -// tree - the tree that is checked -// handle - (optional, default nullptr) - if non-null is set to the type -// -// Return Value: -// Is the tree typeof() -// -bool Compiler::gtIsTypeof(GenTree* tree, CORINFO_CLASS_HANDLE* handle) -{ - if (tree->IsCall()) - { - GenTreeCall* call = tree->AsCall(); - if (gtIsTypeHandleToRuntimeTypeHelper(call)) - { - assert(call->gtArgs.CountArgs() == 1); - CORINFO_CLASS_HANDLE hClass = gtGetHelperArgClassHandle(call->gtArgs.GetArgByIndex(0)->GetEarlyNode()); - if (hClass != NO_CLASS_HANDLE) - { - if (handle != nullptr) - { - *handle = hClass; - } - return true; - } - } - } - if (handle != nullptr) - { - *handle = NO_CLASS_HANDLE; - } - return false; -} - -//------------------------------------------------------------------------ -// gtCallGetDefinedRetBufLclAddr: -// Get the tree corresponding to the address of the retbuf that this call defines. -// -// Parameters: -// call - The call node -// -// Returns: -// A tree representing the address of a local. -// -GenTreeLclVarCommon* Compiler::gtCallGetDefinedRetBufLclAddr(GenTreeCall* call) -{ - if (!call->IsOptimizingRetBufAsLocal()) - { - return nullptr; - } - - CallArg* retBufArg = call->gtArgs.GetRetBufferArg(); - assert(retBufArg != nullptr); - - GenTree* node = retBufArg->GetNode(); - switch (node->OperGet()) - { - // Get the value from putarg wrapper nodes - case GT_PUTARG_REG: - case GT_PUTARG_STK: - node = node->AsOp()->gtGetOp1(); - break; - - default: - break; - } - - // This may be called very late to check validity of LIR. - node = node->gtSkipReloadOrCopy(); - - assert(node->OperIs(GT_LCL_ADDR) && lvaGetDesc(node->AsLclVarCommon())->IsHiddenBufferStructArg()); - - return node->AsLclVarCommon(); -} - -//------------------------------------------------------------------------ -// ParseArrayAddress: Rehydrate the array and index expression from ARR_ADDR. -// -// Arguments: -// comp - The Compiler instance -// pArr - [out] parameter for the tree representing the array instance -// (either an array object pointer, or perhaps a byref to some element) -// pInxVN - [out] parameter for the value number representing the index -// -// Return Value: -// Will set "*pArr" to "nullptr" if this array address is not parseable. -// -// Notes: -// Instead of (or in addition to) parsing the GenTree, maybe we should be parsing the VN -// "trees": if optimization has replaced the index expression with a CSE def, it's harder -// to parse, but the VN tree for the CSE def GT_COMMA has all the same info. For example: -// -// \--* ARR_ADDR byref System.Collections.Hashtable+Bucket[] $80 -// \--* ADD byref -// +--* LCL_VAR ref V01 arg1 u:1 -// \--* COMMA long -// +--* STORE_LCL_VAR long V21 cse2 d:1 -// | \--* ADD long -// | +--* MUL long -// | | +--* CAST long <- uint -// | | | \--* LCL_VAR int V07 loc2 u:2 -// | | \--* CNS_INT long 24 -// | \--* CNS_INT long 16 -// \--* LCL_VAR long V21 cse2 u:1 -// -// Here, the COMMA represents the index + offset VN, and we could pull out the index VN -// from the COMMA VN. -// -void GenTreeArrAddr::ParseArrayAddress(Compiler* comp, GenTree** pArr, ValueNum* pInxVN) -{ - *pArr = nullptr; - ValueNum inxVN = ValueNumStore::NoVN; - target_ssize_t offset = 0; - ParseArrayAddressWork(this->Addr(), comp, 1, pArr, &inxVN, &offset); - - // If we didn't find an array reference (perhaps it is the constant null?) we will give up. - if (*pArr == nullptr) - { - return; - } - - // OK, new we have to figure out if any part of the "offset" is a constant contribution to the index. - target_ssize_t firstElemOffset = GetFirstElemOffset(); - assert(firstElemOffset > 0); - - // If we didn't parse any offset, or the offset we parsed doesn't make sense, then give up on - // parsing the array address. (This can happen with JitOptRepeat.) - if (offset < firstElemOffset) - { - *pArr = nullptr; - return; - } - - unsigned elemSizeUn = (GetElemType() == TYP_STRUCT) ? comp->typGetObjLayout(GetElemClassHandle())->GetSize() - : genTypeSize(GetElemType()); - - assert(FitsIn(elemSizeUn)); - target_ssize_t elemSize = static_cast(elemSizeUn); - target_ssize_t constIndexOffset = offset - firstElemOffset; - - // This should be divisible by the element size... - assert((constIndexOffset % elemSize) == 0); - target_ssize_t constIndex = constIndexOffset / elemSize; - - ValueNumStore* vnStore = comp->GetValueNumStore(); - - if (inxVN == ValueNumStore::NoVN) - { - // Must be a constant index. - *pInxVN = vnStore->VNForPtrSizeIntCon(constIndex); - } - else - { - // - // Perform ((inxVN / elemSizeVN) + vnForConstIndex) - // - - // The value associated with the index value number (inxVN) is the offset into the array, - // which has been scaled by element size. We need to recover the array index from that offset - if (vnStore->IsVNConstant(inxVN)) - { - target_ssize_t index = vnStore->CoercedConstantValue(inxVN); - noway_assert(elemSize > 0 && ((index % elemSize) == 0)); - *pInxVN = vnStore->VNForPtrSizeIntCon((index / elemSize) + constIndex); - } - else - { - bool canFoldDiv = false; - - // If the index VN is a MUL by elemSize, see if we can eliminate it instead of adding - // the division by elemSize. - VNFuncApp funcApp; - if (vnStore->GetVNFunc(inxVN, &funcApp) && funcApp.m_func == (VNFunc)GT_MUL) - { - ValueNum vnForElemSize = vnStore->VNForLongCon(elemSize); - - // One of the multiply operand is elemSize, so the resulting - // index VN should simply be the other operand. - if (funcApp.m_args[1] == vnForElemSize) - { - *pInxVN = funcApp.m_args[0]; - canFoldDiv = true; - } - else if (funcApp.m_args[0] == vnForElemSize) - { - *pInxVN = funcApp.m_args[1]; - canFoldDiv = true; - } - } - - // Perform ((inxVN / elemSizeVN) + vnForConstIndex) - if (!canFoldDiv) - { - ValueNum vnForElemSize = vnStore->VNForPtrSizeIntCon(elemSize); - ValueNum vnForScaledInx = vnStore->VNForFunc(TYP_I_IMPL, VNFunc(GT_DIV), inxVN, vnForElemSize); - *pInxVN = vnForScaledInx; - } - - if (constIndex != 0) - { - ValueNum vnForConstIndex = vnStore->VNForPtrSizeIntCon(constIndex); - - *pInxVN = comp->GetValueNumStore()->VNForFunc(TYP_I_IMPL, VNFunc(GT_ADD), *pInxVN, vnForConstIndex); - } - } - } -} - -/* static */ void GenTreeArrAddr::ParseArrayAddressWork( - GenTree* tree, Compiler* comp, target_ssize_t inputMul, GenTree** pArr, ValueNum* pInxVN, target_ssize_t* pOffset) -{ - if (tree->TypeIs(TYP_REF)) - { - // This must be the array pointer. - assert(*pArr == nullptr); - *pArr = tree; - assert(inputMul == 1); // Can't multiply the array pointer by anything. - } - else - { - switch (tree->OperGet()) - { - case GT_CNS_INT: - assert(!tree->AsIntCon()->ImmedValNeedsReloc(comp)); - // TODO-CrossBitness: we wouldn't need the cast below if GenTreeIntCon::gtIconVal had target_ssize_t - // type. - *pOffset += (inputMul * (target_ssize_t)(tree->AsIntCon()->gtIconVal)); - return; - - case GT_ADD: - case GT_SUB: - ParseArrayAddressWork(tree->AsOp()->gtOp1, comp, inputMul, pArr, pInxVN, pOffset); - if (tree->OperIs(GT_SUB)) - { - inputMul = -inputMul; - } - ParseArrayAddressWork(tree->AsOp()->gtOp2, comp, inputMul, pArr, pInxVN, pOffset); - return; - - case GT_MUL: - { - // If one op is a constant, continue parsing down. - target_ssize_t subMul = 0; - GenTree* nonConst = nullptr; - if (tree->AsOp()->gtOp1->IsCnsIntOrI()) - { - // If the other arg is an int constant, and is a "not-a-field", choose - // that as the multiplier, thus preserving constant index offsets... - if (tree->AsOp()->gtOp2->OperGet() == GT_CNS_INT && - tree->AsOp()->gtOp2->AsIntCon()->gtFieldSeq == nullptr) - { - assert(!tree->AsOp()->gtOp2->AsIntCon()->ImmedValNeedsReloc(comp)); - // TODO-CrossBitness: we wouldn't need the cast below if GenTreeIntConCommon::gtIconVal had - // target_ssize_t type. - subMul = (target_ssize_t)tree->AsOp()->gtOp2->AsIntConCommon()->IconValue(); - nonConst = tree->AsOp()->gtOp1; - } - else - { - assert(!tree->AsOp()->gtOp1->AsIntCon()->ImmedValNeedsReloc(comp)); - // TODO-CrossBitness: we wouldn't need the cast below if GenTreeIntConCommon::gtIconVal had - // target_ssize_t type. - subMul = (target_ssize_t)tree->AsOp()->gtOp1->AsIntConCommon()->IconValue(); - nonConst = tree->AsOp()->gtOp2; - } - } - else if (tree->AsOp()->gtOp2->IsCnsIntOrI()) - { - assert(!tree->AsOp()->gtOp2->AsIntCon()->ImmedValNeedsReloc(comp)); - // TODO-CrossBitness: we wouldn't need the cast below if GenTreeIntConCommon::gtIconVal had - // target_ssize_t type. - subMul = (target_ssize_t)tree->AsOp()->gtOp2->AsIntConCommon()->IconValue(); - nonConst = tree->AsOp()->gtOp1; - } - if (nonConst != nullptr) - { - ParseArrayAddressWork(nonConst, comp, inputMul * subMul, pArr, pInxVN, pOffset); - return; - } - // Otherwise, exit the switch, treat as a contribution to the index. - } - break; - - case GT_LSH: - // If one op is a constant, continue parsing down. - if (tree->AsOp()->gtOp2->IsCnsIntOrI()) - { - assert(!tree->AsOp()->gtOp2->AsIntCon()->ImmedValNeedsReloc(comp)); - // TODO-CrossBitness: we wouldn't need the cast below if GenTreeIntCon::gtIconVal had target_ssize_t - // type. - target_ssize_t shiftVal = (target_ssize_t)tree->AsOp()->gtOp2->AsIntConCommon()->IconValue(); - target_ssize_t subMul = target_ssize_t{1} << shiftVal; - ParseArrayAddressWork(tree->AsOp()->gtOp1, comp, inputMul * subMul, pArr, pInxVN, pOffset); - return; - } - // Otherwise, exit the switch, treat as a contribution to the index. - break; - - case GT_COMMA: - // We don't care about exceptions for this purpose. - if (tree->AsOp()->gtOp1->OperIs(GT_BOUNDS_CHECK) || tree->AsOp()->gtOp1->IsNothingNode()) - { - ParseArrayAddressWork(tree->AsOp()->gtOp2, comp, inputMul, pArr, pInxVN, pOffset); - return; - } - break; - - default: - break; - } - - // If we didn't return above, must be a contribution to the non-constant part of the index VN. - // We don't get here for GT_CNS_INT, GT_ADD, or GT_SUB, or for GT_MUL by constant, or GT_LSH of - // constant shift. Thus, the generated index VN does not include the parsed constant offset. - ValueNum vn = comp->GetValueNumStore()->VNLiberalNormalValue(tree->gtVNPair); - if (inputMul != 1) - { - ValueNum mulVN = comp->GetValueNumStore()->VNForLongCon(inputMul); - vn = comp->GetValueNumStore()->VNForFunc(tree->TypeGet(), VNFunc(GT_MUL), mulVN, vn); - } - if (*pInxVN == ValueNumStore::NoVN) - { - *pInxVN = vn; - } - else - { - *pInxVN = comp->GetValueNumStore()->VNForFunc(tree->TypeGet(), VNFunc(GT_ADD), *pInxVN, vn); - } - } -} - -//------------------------------------------------------------------------ -// IsArrayAddr: Is "this" an expression for an array address? -// -// Recognizes the following patterns: -// this: ARR_ADDR -// this: ADD(ARR_ADDR, CONST) -// -// Arguments: -// pArrAddr - [out] parameter for the found ARR_ADDR node -// -// Return Value: -// Whether "this" matches the pattern denoted above. -// -bool GenTree::IsArrayAddr(GenTreeArrAddr** pArrAddr) -{ - GenTree* addr = this; - if (addr->OperIs(GT_ADD) && addr->AsOp()->gtGetOp2()->IsCnsIntOrI()) - { - addr = addr->AsOp()->gtGetOp1(); - } - - if (addr->OperIs(GT_ARR_ADDR)) - { - *pArrAddr = addr->AsArrAddr(); - return true; - } - - return false; -} - -//------------------------------------------------------------------------ -// SupportsSettingZeroFlag: Returns true if this is an arithmetic operation -// whose codegen supports setting the "zero flag" as part of its operation. -// -// Return Value: -// True if so. A false return does not imply that codegen for the node will -// not trash the zero flag. -// -// Remarks: -// For example, for EQ (AND x y) 0, both xarch and arm64 can emit -// instructions that directly set the flags after the 'AND' and thus no -// comparison is needed. -// -// The backend expects any node for which the flags will be consumed to be -// marked with GTF_SET_FLAGS. -// -bool GenTree::SupportsSettingZeroFlag() -{ -#if defined(TARGET_XARCH) - if (OperIs(GT_AND, GT_OR, GT_XOR, GT_ADD, GT_SUB, GT_NEG)) - { - return true; - } - -#ifdef FEATURE_HW_INTRINSICS - if (OperIs(GT_HWINTRINSIC) && emitter::DoesWriteZeroFlag(HWIntrinsicInfo::lookupIns(AsHWIntrinsic()))) - { - return true; - } -#endif -#elif defined(TARGET_ARM64) - if (OperIs(GT_AND)) - { - return true; - } - - // We do not support setting zero flag for madd/msub. - if (OperIs(GT_ADD, GT_SUB) && (!gtGetOp2()->OperIs(GT_MUL) || !gtGetOp2()->isContained())) - { - return true; - } -#endif - - return false; -} - -//------------------------------------------------------------------------ -// Create: Create or retrieve a field sequence for the given field handle. -// -// The field sequence instance contains some cached information relevant to -// its usage; thus for a given handle all callers of this method must pass -// the same set of arguments. -// -// Arguments: -// fieldHnd - The field handle -// offset - The "offset" value for the field sequence -// fieldKind - The field's kind -// -// Return Value: -// The canonical field sequence for the given field. -// -FieldSeq* FieldSeqStore::Create(CORINFO_FIELD_HANDLE fieldHnd, ssize_t offset, FieldSeq::FieldKind fieldKind) -{ - FieldSeq* fieldSeq = m_map.Emplace(fieldHnd, fieldHnd, offset, fieldKind); - - assert(fieldSeq->GetOffset() == offset); - assert(fieldSeq->GetKind() == fieldKind); - - return fieldSeq; -} - -//------------------------------------------------------------------------ -// Append: "Merge" two field sequences together. -// -// A field sequence only explicitly represents its "head", i. e. the static -// or class field with which it begins. The struct fields that are part of -// it are "implicit" - represented in IR as offsets with "empty" sequences. -// Thus when two sequences are merged, only one can be explicit: -// -// field seq + empty => field seq -// empty + field seq => field seq -// empty + empty => empty -// field seq + field seq => illegal -// -// Arguments: -// a - The field sequence -// b - The second sequence -// -// Return Value: -// The result of "merging" "a" and "b" (see description). -// -FieldSeq* FieldSeqStore::Append(FieldSeq* a, FieldSeq* b) -{ - if (a == nullptr) - { - return b; - } - if (b == nullptr) - { - return a; - } - - // In UB-like code (such as manual IL) we can see an addition of two static field addresses. - // Treat that as cancelling out the sequence, since the result will point nowhere. - // - // It may be possible for the JIT to encounter other types of UB additions, such as due to - // complex optimizations, inlining, etc. In release we'll still do the right thing by returning - // nullptr here, but the more conservative assert can help avoid JIT bugs - - assert(a->GetKind() == FieldSeq::FieldKind::SimpleStaticKnownAddress); - assert(b->GetKind() == FieldSeq::FieldKind::SimpleStaticKnownAddress); - - return nullptr; -} - -FieldSeq::FieldSeq(CORINFO_FIELD_HANDLE fieldHnd, ssize_t offset, FieldKind fieldKind) - : m_offset(offset) -{ - assert(fieldHnd != NO_FIELD_HANDLE); - - uintptr_t handleValue = reinterpret_cast(fieldHnd); - - assert((handleValue & FIELD_KIND_MASK) == 0); - m_fieldHandleAndKind = handleValue | static_cast(fieldKind); - - assert(JitTls::GetCompiler()->eeIsFieldStatic(fieldHnd) == IsStaticField()); - if (fieldKind == FieldKind::Instance) - { - // TODO: enable this assert. At the time of writing, crossgen2 had a bug where the value "getFieldOffset" - // would return for fields with an offset unknown at compile time was incorrect (not zero). - // assert(static_cast(JitTls::GetCompiler()->info.compCompHnd->getFieldOffset(fieldHnd)) == offset); - } -} - -#ifdef FEATURE_SIMD -//------------------------------------------------------------------- -// SetOpLclRelatedToSIMDIntrinsic: Determine if the tree has a local var that needs to be set -// as used by a SIMD intrinsic, and if so, set that local var appropriately. -// -// Arguments: -// op - The tree, to be an operand of a new SIMD-related node, to check. -// -void Compiler::SetOpLclRelatedToSIMDIntrinsic(GenTree* op) -{ - if ((op != nullptr) && op->OperIsScalarLocal()) - { - setLclRelatedToSIMDIntrinsic(op); - } -} - -void GenTreeMultiOp::ResetOperandArray(size_t newOperandCount, - Compiler* compiler, - GenTree** inlineOperands, - size_t inlineOperandCount) -{ - size_t oldOperandCount = GetOperandCount(); - GenTree** oldOperands = GetOperandArray(); - - if (newOperandCount > oldOperandCount) - { - if (newOperandCount <= inlineOperandCount) - { - assert(oldOperandCount <= inlineOperandCount); - assert(oldOperands == inlineOperands); - } - else - { - // The most difficult case: we need to recreate the dynamic array. - assert(compiler != nullptr); - - m_operands = compiler->getAllocator(CMK_ASTNode).allocate(newOperandCount); - } - } - else - { - // We are shrinking the array and may in process switch to an inline representation. - // We choose to do so for simplicity ("if a node has <= InlineOperandCount operands, - // then it stores them inline"), but actually it may be more profitable to not do that, - // it will save us a copy and a potential cache miss (though the latter seems unlikely). - - if ((newOperandCount <= inlineOperandCount) && (oldOperands != inlineOperands)) - { - m_operands = inlineOperands; - } - } - -#ifdef DEBUG - for (size_t i = 0; i < newOperandCount; i++) - { - m_operands[i] = nullptr; - } -#endif // DEBUG - - SetOperandCount(newOperandCount); -} - -/* static */ bool GenTreeMultiOp::OperandsAreEqual(GenTreeMultiOp* op1, GenTreeMultiOp* op2) -{ - if (op1->GetOperandCount() != op2->GetOperandCount()) - { - return false; - } - - for (size_t i = 1; i <= op1->GetOperandCount(); i++) - { - if (!Compare(op1->Op(i), op2->Op(i))) - { - return false; - } - } - - return true; -} - -void GenTreeMultiOp::InitializeOperands(GenTree** operands, size_t operandCount) -{ - for (size_t i = 0; i < operandCount; i++) - { - m_operands[i] = operands[i]; - gtFlags |= (operands[i]->gtFlags & GTF_ALL_EFFECT); - } - - SetOperandCount(operandCount); -} - -//------------------------------------------------------------------------ -// GenTreeJitIntrinsic::SetMethodHandle: Sets the method handle for an intrinsic -// so that it can be rewritten back to a user call in a later phase -// -// Arguments: -// comp - The compiler instance -// methodHandle - The method handle representing the fallback handling for the intrinsic -// entryPoint - The entry point information required for R2R scenarios -// -// Notes: -// We need to ensure that the operands are not tracked inline so that we can track the -// underlying method handle. See the comment in GenTreeJitIntrinsic around why the union -// of fields exists. -// -void GenTreeJitIntrinsic::SetMethodHandle(Compiler* comp, - CORINFO_METHOD_HANDLE methodHandle R2RARG(CORINFO_CONST_LOOKUP entryPoint)) -{ - assert(OperIsHWIntrinsic() && !IsUserCall()); - gtFlags |= (GTF_HW_USER_CALL | GTF_EXCEPT | GTF_CALL); - - size_t operandCount = GetOperandCount(); - - if ((operandCount != 0) && (operandCount <= ArrLen(gtInlineOperands))) - { - GenTree** oldOperands = GetOperandArray(); - GenTree** newOperands = comp->getAllocator(CMK_ASTNode).allocate(operandCount); - - ResetOperandArray(operandCount, comp, newOperands, operandCount); - assert(GetOperandArray() == newOperands); - - for (size_t i = 0; i < operandCount; i++) - { - newOperands[i] = oldOperands[i]; - } - } - - gtMethodHandle = methodHandle; - -#if defined(FEATURE_READYTORUN) - gtEntryPoint = new (comp, CMK_ASTNode) CORINFO_CONST_LOOKUP(entryPoint); -#endif // FEATURE_READYTORUN -} - -var_types GenTreeJitIntrinsic::GetAuxiliaryType() const -{ - CorInfoType auxiliaryJitType = GetAuxiliaryJitType(); - - if (auxiliaryJitType == CORINFO_TYPE_UNDEF) - { - return TYP_UNKNOWN; - } - return JitType2PreciseVarType(auxiliaryJitType); -} - -var_types GenTreeJitIntrinsic::GetSimdBaseType() const -{ - CorInfoType simdBaseJitType = GetSimdBaseJitType(); - - if (simdBaseJitType == CORINFO_TYPE_UNDEF) - { - return TYP_UNKNOWN; - } - return JitType2PreciseVarType(simdBaseJitType); -} -#endif // FEATURE_SIMD - -#ifdef FEATURE_HW_INTRINSICS -//------------------------------------------------------------------------ -// isCommutativeHWIntrinsic: Checks if the intrinsic is commutative -// -// Return Value: -// true if the intrisic is commutative -// -bool GenTree::isCommutativeHWIntrinsic() const -{ - assert(gtOper == GT_HWINTRINSIC); - - const GenTreeHWIntrinsic* node = AsHWIntrinsic(); - NamedIntrinsic id = node->GetHWIntrinsicId(); - - if (HWIntrinsicInfo::IsCommutative(id)) - { - return true; - } - - if (HWIntrinsicInfo::IsMaybeCommutative(id)) - { - switch (id) - { -#ifdef TARGET_XARCH - case NI_SSE_Max: - case NI_SSE_Min: - { - return false; - } - - case NI_SSE2_Max: - case NI_SSE2_Min: - { - return !varTypeIsFloating(node->GetSimdBaseType()); - } - - case NI_AVX_Max: - case NI_AVX_Min: - { - return false; - } - - case NI_AVX512F_Max: - case NI_AVX512F_Min: - { - return !varTypeIsFloating(node->GetSimdBaseType()); - } - - case NI_AVX512F_Add: - case NI_AVX512F_Multiply: - case NI_BMI2_MultiplyNoFlags: - case NI_BMI2_X64_MultiplyNoFlags: - { - return node->GetOperandCount() == 2; - } -#endif // TARGET_XARCH - - default: - { - unreached(); - } - } - } - - return false; -} - -bool GenTree::isContainableHWIntrinsic() const -{ - assert(gtOper == GT_HWINTRINSIC); - -#ifdef TARGET_XARCH - switch (AsHWIntrinsic()->GetHWIntrinsicId()) - { - case NI_SSE_LoadAlignedVector128: - case NI_SSE_LoadScalarVector128: - case NI_SSE2_LoadAlignedVector128: - case NI_SSE2_LoadScalarVector128: - case NI_AVX_LoadAlignedVector256: - case NI_AVX512F_LoadAlignedVector512: - { - // These loads are contained as part of a HWIntrinsic operation - return true; - } - - case NI_AVX512F_ConvertToVector256Int32: - case NI_AVX512F_ConvertToVector256UInt32: - case NI_AVX512F_VL_ConvertToVector128UInt32: - case NI_AVX512F_VL_ConvertToVector128UInt32WithSaturation: - case NI_AVX10v1_ConvertToVector128UInt32: - case NI_AVX10v1_ConvertToVector128UInt32WithSaturation: - { - if (varTypeIsFloating(AsHWIntrinsic()->GetSimdBaseType())) - { - return false; - } - FALLTHROUGH; - } - - case NI_Vector128_GetElement: - case NI_Vector128_ToScalar: - case NI_Vector256_ToScalar: - case NI_Vector512_ToScalar: - case NI_SSE2_ConvertToInt32: - case NI_SSE2_ConvertToUInt32: - case NI_SSE2_X64_ConvertToInt64: - case NI_SSE2_X64_ConvertToUInt64: - case NI_SSE2_Extract: - case NI_SSE41_Extract: - case NI_SSE41_X64_Extract: - case NI_AVX_ExtractVector128: - case NI_AVX2_ConvertToInt32: - case NI_AVX2_ConvertToUInt32: - case NI_AVX2_ExtractVector128: - case NI_AVX512F_ExtractVector128: - case NI_AVX512F_ExtractVector256: - case NI_AVX512F_ConvertToVector128Byte: - case NI_AVX512F_ConvertToVector128ByteWithSaturation: - case NI_AVX512F_ConvertToVector128Int16: - case NI_AVX512F_ConvertToVector128Int16WithSaturation: - case NI_AVX512F_ConvertToVector128SByte: - case NI_AVX512F_ConvertToVector128SByteWithSaturation: - case NI_AVX512F_ConvertToVector128UInt16: - case NI_AVX512F_ConvertToVector128UInt16WithSaturation: - case NI_AVX512F_ConvertToVector256Int16: - case NI_AVX512F_ConvertToVector256Int16WithSaturation: - case NI_AVX512F_ConvertToVector256Int32WithSaturation: - case NI_AVX512F_ConvertToVector256UInt16: - case NI_AVX512F_ConvertToVector256UInt16WithSaturation: - case NI_AVX512F_ConvertToVector256UInt32WithSaturation: - case NI_AVX512F_VL_ConvertToVector128Byte: - case NI_AVX512F_VL_ConvertToVector128ByteWithSaturation: - case NI_AVX512F_VL_ConvertToVector128Int16: - case NI_AVX512F_VL_ConvertToVector128Int16WithSaturation: - case NI_AVX512F_VL_ConvertToVector128Int32: - case NI_AVX512F_VL_ConvertToVector128Int32WithSaturation: - case NI_AVX512F_VL_ConvertToVector128SByte: - case NI_AVX512F_VL_ConvertToVector128SByteWithSaturation: - case NI_AVX512F_VL_ConvertToVector128UInt16: - case NI_AVX512F_VL_ConvertToVector128UInt16WithSaturation: - case NI_AVX512BW_ConvertToVector256Byte: - case NI_AVX512BW_ConvertToVector256ByteWithSaturation: - case NI_AVX512BW_ConvertToVector256SByte: - case NI_AVX512BW_ConvertToVector256SByteWithSaturation: - case NI_AVX512BW_VL_ConvertToVector128Byte: - case NI_AVX512BW_VL_ConvertToVector128ByteWithSaturation: - case NI_AVX512BW_VL_ConvertToVector128SByte: - case NI_AVX512BW_VL_ConvertToVector128SByteWithSaturation: - case NI_AVX512DQ_ExtractVector128: - case NI_AVX512DQ_ExtractVector256: - case NI_AVX10v1_ConvertToVector128Byte: - case NI_AVX10v1_ConvertToVector128ByteWithSaturation: - case NI_AVX10v1_ConvertToVector128Int16: - case NI_AVX10v1_ConvertToVector128Int16WithSaturation: - case NI_AVX10v1_ConvertToVector128Int32: - case NI_AVX10v1_ConvertToVector128Int32WithSaturation: - case NI_AVX10v1_ConvertToVector128SByte: - case NI_AVX10v1_ConvertToVector128SByteWithSaturation: - case NI_AVX10v1_ConvertToVector128UInt16: - case NI_AVX10v1_ConvertToVector128UInt16WithSaturation: - case NI_AVX10v1_V512_ExtractVector128: - case NI_AVX10v1_V512_ExtractVector256: - { - // These HWIntrinsic operations are contained as part of a store - return true; - } - - case NI_Vector128_CreateScalarUnsafe: - case NI_Vector256_CreateScalarUnsafe: - case NI_Vector512_CreateScalarUnsafe: - { - // These HWIntrinsic operations are contained as part of scalar ops - return true; - } - - case NI_SSE3_LoadAndDuplicateToVector128: - case NI_SSE3_MoveAndDuplicate: - case NI_AVX_BroadcastScalarToVector128: - case NI_AVX2_BroadcastScalarToVector128: - case NI_AVX_BroadcastScalarToVector256: - case NI_AVX2_BroadcastScalarToVector256: - case NI_AVX512F_BroadcastScalarToVector512: - { - // These intrinsic operations are contained as part of the operand of embedded broadcast compatible - // instruction - return true; - } - - default: - { - return false; - } - } -#elif defined(TARGET_ARM64) - return (AsHWIntrinsic()->GetHWIntrinsicId() == NI_Sve_ConditionalSelect); -#else - return false; -#endif // TARGET_XARCH -} - -bool GenTree::isRMWHWIntrinsic(Compiler* comp) -{ - assert(gtOper == GT_HWINTRINSIC); - assert(comp != nullptr); - -#if defined(TARGET_XARCH) - GenTreeHWIntrinsic* hwintrinsic = AsHWIntrinsic(); - NamedIntrinsic intrinsicId = hwintrinsic->GetHWIntrinsicId(); - - if (!comp->canUseVexEncoding()) - { - return HWIntrinsicInfo::HasRMWSemantics(intrinsicId); - } - - if (HWIntrinsicInfo::IsRmwIntrinsic(intrinsicId)) - { - return true; - } - - switch (intrinsicId) - { - case NI_EVEX_BlendVariableMask: - { - GenTree* op2 = hwintrinsic->Op(2); - - if (op2->IsEmbMaskOp()) - { - GenTree* op1 = hwintrinsic->Op(1); - - if (op1->isContained()) - { - assert(op1->IsVectorZero()); - return false; - } - else - { - return true; - } - } - - return false; - } - - case NI_AVX512F_Fixup: - case NI_AVX512F_FixupScalar: - case NI_AVX512F_VL_Fixup: - case NI_AVX10v1_Fixup: - case NI_AVX10v1_FixupScalar: - { - // We are actually only RMW in the case where the lookup table - // has any value that could result in `op1` being picked. So - // in the case `op3` is a constant and none of the nibbles are - // `0`, then we don't have to be RMW and can actually "drop" `op1` - - GenTree* op4 = hwintrinsic->Op(4); - - if (!op4->IsIntegralConst()) - { - return true; - } - - GenTree* op3 = hwintrinsic->Op(3); - - if (!op3->IsCnsVec()) - { - return true; - } - - GenTreeVecCon* vecCon = op3->AsVecCon(); - - var_types simdBaseType = hwintrinsic->GetSimdBaseType(); - unsigned simdSize = hwintrinsic->GetSimdSize(); - uint32_t count = simdSize / sizeof(uint32_t); - uint32_t incSize = (simdBaseType == TYP_FLOAT) ? 1 : 2; - - if (intrinsicId == NI_AVX512F_FixupScalar || intrinsicId == NI_AVX10v1_FixupScalar) - { - // Upper elements come from op2 - count = 1; - } - - for (uint32_t i = 0; i < count; i += incSize) - { - uint32_t tbl = vecCon->gtSimdVal.u32[i]; - - if (((tbl & 0x0000000F) == 0) || ((tbl & 0x000000F0) == 0) || ((tbl & 0x00000F00) == 0) || - ((tbl & 0x0000F000) == 0) || ((tbl & 0x000F0000) == 0) || ((tbl & 0x00F00000) == 0) || - ((tbl & 0x0F000000) == 0) || ((tbl & 0xF0000000) == 0)) - { - return true; - } - } - - return false; - } - - case NI_AVX512F_TernaryLogic: - case NI_AVX512F_VL_TernaryLogic: - case NI_AVX10v1_TernaryLogic: - { - // We may not be RMW depending on the control byte as there - // are many operations that do not use all three inputs. - - GenTree* op4 = hwintrinsic->Op(4); - - if (!op4->IsIntegralConst()) - { - return true; - } - - uint8_t control = static_cast(op4->AsIntCon()->gtIconVal); - const TernaryLogicInfo& info = TernaryLogicInfo::lookup(control); - TernaryLogicUseFlags useFlags = info.GetAllUseFlags(); - - return useFlags == TernaryLogicUseFlags::ABC; - } - - default: - { - return false; - } - } -#elif defined(TARGET_ARM64) - return HWIntrinsicInfo::HasRMWSemantics(AsHWIntrinsic()->GetHWIntrinsicId()); -#else - return false; -#endif -} - -//------------------------------------------------------------------------ -// isEvexCompatibleHWIntrinsic: Checks if the intrinsic has a compatible -// EVEX form for its intended lowering instruction. -// -// Return Value: -// true if the intrinsic node lowering instruction has an EVEX form -// -bool GenTree::isEvexCompatibleHWIntrinsic(Compiler* comp) const -{ -#if defined(TARGET_XARCH) - if (OperIsHWIntrinsic()) - { - NamedIntrinsic intrinsicId = AsHWIntrinsic()->GetHWIntrinsicId(); - - switch (intrinsicId) - { - case NI_PCLMULQDQ_CarrylessMultiply: - { - return comp->compOpportunisticallyDependsOn(InstructionSet_PCLMULQDQ_V256); - } - - default: - { - return HWIntrinsicInfo::HasEvexSemantics(intrinsicId); - } - } - } -#endif - return false; -} - -//------------------------------------------------------------------------ -// isEmbeddedMaskingCompatibleHWIntrinsic : Checks if the intrinsic is compatible -// with the EVEX embedded masking form for its intended lowering instruction. -// -// Return Value: -// true if the intrinsic node lowering instruction has an EVEX embedded masking -// -bool GenTree::isEmbeddedMaskingCompatibleHWIntrinsic() const -{ - if (OperIsHWIntrinsic()) - { - NamedIntrinsic intrinsicId = AsHWIntrinsic()->GetHWIntrinsicId(); -#if defined(TARGET_XARCH) - var_types simdBaseType = AsHWIntrinsic()->GetSimdBaseType(); - - switch (intrinsicId) - { - case NI_AVX512F_ConvertToVector256Int32: - case NI_AVX512F_ConvertToVector256UInt32: - case NI_AVX512F_ConvertToVector512Int32: - case NI_AVX512F_ConvertToVector512UInt32: - case NI_AVX512F_VL_ConvertToVector128UInt32: - case NI_AVX10v1_ConvertToVector128UInt32: - { - return varTypeIsFloating(simdBaseType); - } - - default: - { - return HWIntrinsicInfo::IsEmbMaskingCompatible(intrinsicId); - } - } -#elif defined(TARGET_ARM64) - return HWIntrinsicInfo::IsEmbeddedMaskedOperation(intrinsicId) || - HWIntrinsicInfo::IsOptionalEmbeddedMaskedOperation(intrinsicId); -#endif - } - return false; -} - -GenTreeHWIntrinsic* Compiler::gtNewSimdHWIntrinsicNode(var_types type, - NamedIntrinsic hwIntrinsicID, - CorInfoType simdBaseJitType, - unsigned simdSize) -{ - return new (this, GT_HWINTRINSIC) - GenTreeHWIntrinsic(type, getAllocator(CMK_ASTNode), hwIntrinsicID, simdBaseJitType, simdSize); -} - -GenTreeHWIntrinsic* Compiler::gtNewSimdHWIntrinsicNode( - var_types type, GenTree* op1, NamedIntrinsic hwIntrinsicID, CorInfoType simdBaseJitType, unsigned simdSize) -{ - SetOpLclRelatedToSIMDIntrinsic(op1); - - return new (this, GT_HWINTRINSIC) - GenTreeHWIntrinsic(type, getAllocator(CMK_ASTNode), hwIntrinsicID, simdBaseJitType, simdSize, op1); -} - -GenTreeHWIntrinsic* Compiler::gtNewSimdHWIntrinsicNode(var_types type, - GenTree* op1, - GenTree* op2, - NamedIntrinsic hwIntrinsicID, - CorInfoType simdBaseJitType, - unsigned simdSize) -{ - SetOpLclRelatedToSIMDIntrinsic(op1); - SetOpLclRelatedToSIMDIntrinsic(op2); - - return new (this, GT_HWINTRINSIC) - GenTreeHWIntrinsic(type, getAllocator(CMK_ASTNode), hwIntrinsicID, simdBaseJitType, simdSize, op1, op2); -} - -GenTreeHWIntrinsic* Compiler::gtNewSimdHWIntrinsicNode(var_types type, - GenTree* op1, - GenTree* op2, - GenTree* op3, - NamedIntrinsic hwIntrinsicID, - CorInfoType simdBaseJitType, - unsigned simdSize) -{ - SetOpLclRelatedToSIMDIntrinsic(op1); - SetOpLclRelatedToSIMDIntrinsic(op2); - SetOpLclRelatedToSIMDIntrinsic(op3); - - return new (this, GT_HWINTRINSIC) - GenTreeHWIntrinsic(type, getAllocator(CMK_ASTNode), hwIntrinsicID, simdBaseJitType, simdSize, op1, op2, op3); -} - -GenTreeHWIntrinsic* Compiler::gtNewSimdHWIntrinsicNode(var_types type, - GenTree* op1, - GenTree* op2, - GenTree* op3, - GenTree* op4, - NamedIntrinsic hwIntrinsicID, - CorInfoType simdBaseJitType, - unsigned simdSize) -{ - SetOpLclRelatedToSIMDIntrinsic(op1); - SetOpLclRelatedToSIMDIntrinsic(op2); - SetOpLclRelatedToSIMDIntrinsic(op3); - SetOpLclRelatedToSIMDIntrinsic(op4); - - return new (this, GT_HWINTRINSIC) GenTreeHWIntrinsic(type, getAllocator(CMK_ASTNode), hwIntrinsicID, - simdBaseJitType, simdSize, op1, op2, op3, op4); -} - -GenTreeHWIntrinsic* Compiler::gtNewSimdHWIntrinsicNode(var_types type, - GenTree** operands, - size_t operandCount, - NamedIntrinsic hwIntrinsicID, - CorInfoType simdBaseJitType, - unsigned simdSize) -{ - IntrinsicNodeBuilder nodeBuilder(getAllocator(CMK_ASTNode), operandCount); - for (size_t i = 0; i < operandCount; i++) - { - nodeBuilder.AddOperand(i, operands[i]); - SetOpLclRelatedToSIMDIntrinsic(operands[i]); - } - - return new (this, GT_HWINTRINSIC) - GenTreeHWIntrinsic(type, std::move(nodeBuilder), hwIntrinsicID, simdBaseJitType, simdSize); -} - -GenTreeHWIntrinsic* Compiler::gtNewSimdHWIntrinsicNode(var_types type, - IntrinsicNodeBuilder&& nodeBuilder, - NamedIntrinsic hwIntrinsicID, - CorInfoType simdBaseJitType, - unsigned simdSize) -{ - for (size_t i = 0; i < nodeBuilder.GetOperandCount(); i++) - { - SetOpLclRelatedToSIMDIntrinsic(nodeBuilder.GetOperand(i)); - } - - return new (this, GT_HWINTRINSIC) - GenTreeHWIntrinsic(type, std::move(nodeBuilder), hwIntrinsicID, simdBaseJitType, simdSize); -} - -GenTree* Compiler::gtNewSimdAbsNode(var_types type, GenTree* op1, CorInfoType simdBaseJitType, unsigned simdSize) -{ - assert(IsBaselineSimdIsaSupportedDebugOnly()); - - assert(varTypeIsSIMD(type)); - assert(getSIMDTypeForSize(simdSize) == type); - - assert(op1 != nullptr); - assert(op1->TypeGet() == type); - - var_types simdBaseType = JitType2PreciseVarType(simdBaseJitType); - assert(varTypeIsArithmetic(simdBaseType)); - - if (varTypeIsUnsigned(simdBaseType)) - { - return op1; - } - -#if defined(TARGET_XARCH) - if (varTypeIsFloating(simdBaseType)) - { - // Abs(v) = v & ~new vector(-0.0); - assert((simdSize != 32) || compIsaSupportedDebugOnly(InstructionSet_AVX)); - - GenTree* bitMask; - - if (simdBaseType == TYP_FLOAT) - { - bitMask = gtNewIconNode(0x7FFFFFFF); - bitMask = gtNewSimdCreateBroadcastNode(type, bitMask, CORINFO_TYPE_INT, simdSize); - } - else - { - bitMask = gtNewLconNode(0x7FFFFFFFFFFFFFFF); - bitMask = gtNewSimdCreateBroadcastNode(type, bitMask, CORINFO_TYPE_LONG, simdSize); - } - return gtNewSimdBinOpNode(GT_AND, type, op1, bitMask, simdBaseJitType, simdSize); - } - - NamedIntrinsic intrinsic = NI_Illegal; - - if (simdBaseType == TYP_LONG) - { - if (simdSize == 64) - { - assert(compIsaSupportedDebugOnly(InstructionSet_AVX512F)); - intrinsic = NI_AVX512F_Abs; - } - else if (compOpportunisticallyDependsOn(InstructionSet_AVX10v1)) - { - intrinsic = NI_AVX10v1_Abs; - } - else if (compOpportunisticallyDependsOn(InstructionSet_AVX512F_VL)) - { - intrinsic = NI_AVX512F_VL_Abs; - } - } - else if (simdSize == 32) - { - assert(compIsaSupportedDebugOnly(InstructionSet_AVX2)); - intrinsic = NI_AVX2_Abs; - } - else if (simdSize == 64) - { - if (simdBaseType == TYP_INT) - { - assert(compIsaSupportedDebugOnly(InstructionSet_AVX512F)); - intrinsic = NI_AVX512F_Abs; - } - else - { - assert(varTypeIsSmall(simdBaseType)); - assert(compIsaSupportedDebugOnly(InstructionSet_AVX512BW)); - intrinsic = NI_AVX512BW_Abs; - } - } - else if (compOpportunisticallyDependsOn(InstructionSet_SSSE3)) - { - intrinsic = NI_SSSE3_Abs; - } - - if (intrinsic != NI_Illegal) - { - return gtNewSimdHWIntrinsicNode(type, op1, intrinsic, simdBaseJitType, simdSize); - } - else - { - GenTree* tmp; - GenTree* op1Dup1 = fgMakeMultiUse(&op1); - GenTree* op1Dup2 = gtCloneExpr(op1Dup1); - - // op1 = IsNegative(op1) - op1 = gtNewSimdIsNegativeNode(type, op1, simdBaseJitType, simdSize); - - // tmp = -op1Dup1 - tmp = gtNewSimdUnOpNode(GT_NEG, type, op1Dup1, simdBaseJitType, simdSize); - - // result = ConditionalSelect(op1, tmp, op1Dup2) - return gtNewSimdCndSelNode(type, op1, tmp, op1Dup2, simdBaseJitType, simdSize); - } -#elif defined(TARGET_ARM64) - NamedIntrinsic intrinsic = NI_AdvSimd_Abs; - - if (simdBaseType == TYP_DOUBLE) - { - intrinsic = (simdSize == 8) ? NI_AdvSimd_AbsScalar : NI_AdvSimd_Arm64_Abs; - } - else if (varTypeIsLong(simdBaseType)) - { - intrinsic = (simdSize == 8) ? NI_AdvSimd_Arm64_AbsScalar : NI_AdvSimd_Arm64_Abs; - } - - assert(intrinsic != NI_Illegal); - return gtNewSimdHWIntrinsicNode(type, op1, intrinsic, simdBaseJitType, simdSize); -#else -#error Unsupported platform -#endif -} - -GenTree* Compiler::gtNewSimdBinOpNode( - genTreeOps op, var_types type, GenTree* op1, GenTree* op2, CorInfoType simdBaseJitType, unsigned simdSize) -{ - assert(IsBaselineSimdIsaSupportedDebugOnly()); - - assert(varTypeIsSIMD(type)); - assert(getSIMDTypeForSize(simdSize) == type); - - var_types simdBaseType = JitType2PreciseVarType(simdBaseJitType); - assert(varTypeIsArithmetic(simdBaseType)); - - assert(op1 != nullptr); - assert(op1->TypeIs(type, simdBaseType, genActualType(simdBaseType)) || - (op1->TypeIs(TYP_SIMD12) && type == TYP_SIMD16)); - - assert(op2 != nullptr); - - if ((op == GT_LSH) || (op == GT_RSH) || (op == GT_RSZ)) - { - assert(genActualType(op2) == TYP_INT); - } - else - { - assert((genActualType(op2) == genActualType(type)) || (genActualType(op2) == genActualType(simdBaseType)) || - (op2->TypeIs(TYP_SIMD12) && (type == TYP_SIMD16))); - } - - bool needsReverseOps = false; - GenTree* op2ForLookup = nullptr; - - switch (op) - { - case GT_DIV: - { - if (varTypeIsArithmetic(op2)) - { - op2 = gtNewSimdCreateBroadcastNode(type, op2, simdBaseJitType, simdSize); - } - break; - } - - case GT_LSH: - case GT_RSH: - case GT_RSZ: - { - // float and double don't have actual instructions for shifting - // so we'll just use the equivalent integer instruction instead. - - if (simdBaseType == TYP_FLOAT) - { - simdBaseJitType = CORINFO_TYPE_INT; - simdBaseType = TYP_INT; - } - else if (simdBaseType == TYP_DOUBLE) - { - simdBaseJitType = CORINFO_TYPE_LONG; - simdBaseType = TYP_LONG; - } - - // "over shifting" is platform specific behavior. We will match the C# behavior - // this requires we mask with (sizeof(T) * 8) - 1 which ensures the shift cannot - // exceed the number of bits available in `T`. This is roughly equivalent to - // x % (sizeof(T) * 8), but that is "more expensive" and only the same for unsigned - // inputs, where-as we have a signed-input and so negative values would differ. - - unsigned shiftCountMask = (genTypeSize(simdBaseType) * 8) - 1; - - if (op2->IsCnsIntOrI()) - { - op2->AsIntCon()->gtIconVal &= shiftCountMask; -#ifdef TARGET_ARM64 - // On ARM64, ShiftRight* intrinsics cannot encode a shift value of zero, - // so use the generic Shift* fallback intrinsic. - // GenTreeHWIntrinsic::GetHWIntrinsicIdForBinOp will see that the immediate node is not const, - // and return the correct fallback intrinsic. - if ((op != GT_LSH) && (op2->AsIntCon()->IconValue() == 0)) - { - op2 = gtNewZeroConNode(type); - } -#endif // TARGET_ARM64 - } - else - { - op2 = gtNewOperNode(GT_AND, TYP_INT, op2, gtNewIconNode(shiftCountMask)); - -#if defined(TARGET_XARCH) - op2ForLookup = op2; - op2 = gtNewSimdCreateScalarNode(TYP_SIMD16, op2, CORINFO_TYPE_INT, 16); -#elif defined(TARGET_ARM64) - if (op != GT_LSH) - { - op2 = gtNewOperNode(GT_NEG, TYP_INT, op2); - } - - op2 = gtNewSimdCreateBroadcastNode(type, op2, simdBaseJitType, simdSize); -#endif // !TARGET_XARCH && !TARGET_ARM64 - } - break; - } - - case GT_MUL: - { - GenTree** broadcastOp = nullptr; - - if (varTypeIsArithmetic(op1)) - { - broadcastOp = &op1; - -#if defined(TARGET_ARM64) - if (!varTypeIsByte(simdBaseType)) - { - // MultiplyByScalar requires the scalar op to be op2 for GetHWIntrinsicIdForBinOp - needsReverseOps = true; - } -#endif // TARGET_ARM64 - } - else if (varTypeIsArithmetic(op2)) - { - broadcastOp = &op2; - } - - if (broadcastOp != nullptr) - { -#if defined(TARGET_ARM64) - if (varTypeIsLong(simdBaseType)) - { - // This is handled via emulation and the scalar is consumed directly - break; - } - else if (!varTypeIsByte(simdBaseType)) - { - op2ForLookup = *broadcastOp; - *broadcastOp = gtNewSimdCreateScalarUnsafeNode(TYP_SIMD8, *broadcastOp, simdBaseJitType, 8); - break; - } -#endif // TARGET_ARM64 - - *broadcastOp = gtNewSimdCreateBroadcastNode(type, *broadcastOp, simdBaseJitType, simdSize); - } - break; - } - -#if defined(TARGET_XARCH) - case GT_AND: - case GT_AND_NOT: - case GT_OR: - case GT_XOR: - { - if (simdSize == 32) - { - if (varTypeIsIntegral(simdBaseType) && !compOpportunisticallyDependsOn(InstructionSet_AVX2)) - { - if (varTypeIsLong(simdBaseType)) - { - simdBaseJitType = CORINFO_TYPE_DOUBLE; - simdBaseType = TYP_DOUBLE; - } - else - { - simdBaseJitType = CORINFO_TYPE_FLOAT; - simdBaseType = TYP_FLOAT; - } - } - } - break; - } -#endif // TARGET_XARCH - - default: - { - break; - } - } - - if (needsReverseOps) - { - // We expect op1 to have already been spilled if needed - std::swap(op1, op2); - } - - if (op2ForLookup == nullptr) - { - op2ForLookup = op2; - } - else - { - assert(op2ForLookup != op1); - } - - NamedIntrinsic intrinsic = - GenTreeHWIntrinsic::GetHWIntrinsicIdForBinOp(this, op, op1, op2ForLookup, simdBaseType, simdSize, false); - - if (intrinsic != NI_Illegal) - { - if (op == GT_AND_NOT) - { - assert(fgNodeThreading == NodeThreading::LIR); - -#if defined(TARGET_XARCH) - // GT_AND_NOT expects `op1 & ~op2`, but xarch does `~op1 & op2` - // We specially handle this here since we're only producing a - // native intrinsic node in LIR - - std::swap(op1, op2); -#endif // TARGET_XARCH - } -#ifdef TARGET_XARCH - if (HWIntrinsicInfo::NeedsNormalizeSmallTypeToInt(intrinsic) && varTypeIsSmall(simdBaseType)) - { - simdBaseJitType = varTypeIsUnsigned(simdBaseType) ? CORINFO_TYPE_UINT : CORINFO_TYPE_INT; - simdBaseType = JitType2PreciseVarType(simdBaseJitType); - } -#endif // TARGET_XARCH - return gtNewSimdHWIntrinsicNode(type, op1, op2, intrinsic, simdBaseJitType, simdSize); - } - - switch (op) - { - case GT_AND_NOT: - { - // Prior to LIR, we want to explicitly decompose this operation so that downstream phases can - // appropriately optimize around the individual operations being performed, particularly ~op2, - // and produce overall better codegen. - assert(fgNodeThreading != NodeThreading::LIR); - - op2 = gtNewSimdUnOpNode(GT_NOT, type, op2, simdBaseJitType, simdSize); - return gtNewSimdBinOpNode(GT_AND, type, op1, op2, simdBaseJitType, simdSize); - } - -#if defined(TARGET_XARCH) - case GT_LSH: - case GT_RSH: - case GT_RSZ: - { - // This emulates byte shift instructions, which don't exist in x86 SIMD, - // plus arithmetic shift of qwords, which did not exist before AVX-512. - - assert(varTypeIsByte(simdBaseType) || (varTypeIsLong(simdBaseType) && (op == GT_RSH))); - - // We will emulate arithmetic shift by using logical shift and then masking in the sign bits. - genTreeOps instrOp = op == GT_RSH ? GT_RSZ : op; - intrinsic = GenTreeHWIntrinsic::GetHWIntrinsicIdForBinOp(this, instrOp, op1, op2ForLookup, - genActualType(simdBaseType), simdSize, false); - assert(intrinsic != NI_Illegal); - - GenTree* maskAmountOp; - - if (op2->IsCnsIntOrI()) - { - ssize_t shiftCount = op2->AsIntCon()->gtIconVal; - if (varTypeIsByte(simdBaseType)) - { - ssize_t mask = op == GT_LSH ? ((0xFF << shiftCount) & 0xFF) : (0xFF >> shiftCount); - maskAmountOp = gtNewIconNode(mask, type); - } - else - { - int64_t mask = static_cast(0xFFFFFFFFFFFFFFFFULL >> shiftCount); - maskAmountOp = gtNewLconNode(mask); - } - } - else - { - assert(op2->OperIsHWIntrinsic(NI_Vector128_CreateScalar)); - - GenTree* shiftCountDup = fgMakeMultiUse(&op2->AsHWIntrinsic()->Op(1)); - if (op == GT_RSH) - { - // For arithmetic shift, we will be using ConditionalSelect to mask in the sign bits, which means - // the mask will be evaluated before the shift. We swap the copied operand with the shift amount - // operand here in order to preserve correct evaluation order for the masked shift count. - std::swap(shiftCountDup, op2->AsHWIntrinsic()->Op(1)); - } - - maskAmountOp = gtNewOperNode(instrOp, genActualType(simdBaseType), gtNewAllBitsSetConNode(simdBaseType), - shiftCountDup); - } - - if (op == GT_RSH) - { - GenTree* op1Dup = fgMakeMultiUse(&op1); - GenTree* signOp = - gtNewSimdCmpOpNode(GT_GT, type, gtNewZeroConNode(type), op1Dup, simdBaseJitType, simdSize); - - CorInfoType shiftType = varTypeIsSmall(simdBaseType) ? CORINFO_TYPE_INT : simdBaseJitType; - GenTree* shiftOp = gtNewSimdHWIntrinsicNode(type, op1, op2, intrinsic, shiftType, simdSize); - GenTree* maskOp = gtNewSimdCreateBroadcastNode(type, maskAmountOp, simdBaseJitType, simdSize); - - return gtNewSimdCndSelNode(type, maskOp, shiftOp, signOp, simdBaseJitType, simdSize); - } - else - { - GenTree* shiftOp = gtNewSimdHWIntrinsicNode(type, op1, op2, intrinsic, CORINFO_TYPE_INT, simdSize); - GenTree* maskOp = gtNewSimdCreateBroadcastNode(type, maskAmountOp, simdBaseJitType, simdSize); - - return gtNewSimdBinOpNode(GT_AND, type, shiftOp, maskOp, simdBaseJitType, simdSize); - } - } -#endif // TARGET_XARCH - - case GT_MUL: - { -#if defined(TARGET_XARCH) - if (varTypeIsByte(simdBaseType)) - { - assert((simdSize != 64) || IsBaselineVector512IsaSupportedDebugOnly()); - - CorInfoType widenedSimdBaseJitType; - NamedIntrinsic widenIntrinsic; - NamedIntrinsic narrowIntrinsic; - var_types widenedType; - unsigned widenedSimdSize; - bool isV512Supported = false; - - if (simdSize == 32 && IsBaselineVector512IsaSupportedOpportunistically()) - { - // Input is SIMD32 [U]Byte and AVX512BW is supported: - // - Widen inputs as SIMD64 [U]Short - // - Multiply widened inputs (SIMD64 [U]Short) as widened product (SIMD64 [U]Short) - // - Narrow widened product (SIMD64 [U]Short) as SIMD32 [U]Byte - if (simdBaseType == TYP_BYTE) - { - widenedSimdBaseJitType = CORINFO_TYPE_SHORT; - widenIntrinsic = NI_AVX512BW_ConvertToVector512Int16; - narrowIntrinsic = NI_AVX512BW_ConvertToVector256SByte; - } - else - { - widenedSimdBaseJitType = CORINFO_TYPE_USHORT; - widenIntrinsic = NI_AVX512BW_ConvertToVector512UInt16; - narrowIntrinsic = NI_AVX512BW_ConvertToVector256Byte; - } - - widenedType = TYP_SIMD64; - widenedSimdSize = 64; - - // Vector512 widenedOp1 = Avx512BW.ConvertToVector512UInt16(op1) - GenTree* widenedOp1 = - gtNewSimdHWIntrinsicNode(widenedType, op1, widenIntrinsic, simdBaseJitType, widenedSimdSize); - - // Vector512 widenedOp2 = Avx512BW.ConvertToVector512UInt16(op2) - GenTree* widenedOp2 = - gtNewSimdHWIntrinsicNode(widenedType, op2, widenIntrinsic, simdBaseJitType, widenedSimdSize); - - // Vector512 widenedProduct = widenedOp1 * widenedOp2; - GenTree* widenedProduct = gtNewSimdBinOpNode(GT_MUL, widenedType, widenedOp1, widenedOp2, - widenedSimdBaseJitType, widenedSimdSize); - - // Vector256 product = Avx512BW.ConvertToVector256Byte(widenedProduct) - return gtNewSimdHWIntrinsicNode(type, widenedProduct, narrowIntrinsic, widenedSimdBaseJitType, - widenedSimdSize); - } - else if (simdSize == 16 && compOpportunisticallyDependsOn(InstructionSet_AVX2)) - { - if (compIsEvexOpportunisticallySupported(isV512Supported)) - { - // Input is SIMD16 [U]Byte and AVX512BW_VL is supported: - // - Widen inputs as SIMD32 [U]Short - // - Multiply widened inputs (SIMD32 [U]Short) as widened product (SIMD32 [U]Short) - // - Narrow widened product (SIMD32 [U]Short) as SIMD16 [U]Byte - widenIntrinsic = NI_AVX2_ConvertToVector256Int16; - - if (simdBaseType == TYP_BYTE) - { - widenedSimdBaseJitType = CORINFO_TYPE_SHORT; - narrowIntrinsic = !isV512Supported ? NI_AVX10v1_ConvertToVector128SByte - : NI_AVX512BW_VL_ConvertToVector128SByte; - } - else - { - widenedSimdBaseJitType = CORINFO_TYPE_USHORT; - narrowIntrinsic = !isV512Supported ? NI_AVX10v1_ConvertToVector128Byte - : NI_AVX512BW_VL_ConvertToVector128Byte; - } - - widenedType = TYP_SIMD32; - widenedSimdSize = 32; - - // Vector256 widenedOp1 = Avx2.ConvertToVector256Int16(op1).AsUInt16() - GenTree* widenedOp1 = gtNewSimdHWIntrinsicNode(widenedType, op1, widenIntrinsic, - simdBaseJitType, widenedSimdSize); - - // Vector256 widenedOp2 = Avx2.ConvertToVector256Int16(op2).AsUInt16() - GenTree* widenedOp2 = gtNewSimdHWIntrinsicNode(widenedType, op2, widenIntrinsic, - simdBaseJitType, widenedSimdSize); - - // Vector256 widenedProduct = widenedOp1 * widenedOp2 - GenTree* widenedProduct = gtNewSimdBinOpNode(GT_MUL, widenedType, widenedOp1, widenedOp2, - widenedSimdBaseJitType, widenedSimdSize); - - // Vector128 product = Avx512BW.VL.ConvertToVector128Byte(widenedProduct) - return gtNewSimdHWIntrinsicNode(type, widenedProduct, narrowIntrinsic, widenedSimdBaseJitType, - widenedSimdSize); - } - else - { - // Input is SIMD16 [U]Byte and AVX512BW_VL is NOT supported (only AVX2 will be used): - // - Widen inputs as SIMD32 [U]Short - // - Multiply widened inputs (SIMD32 [U]Short) as widened product (SIMD32 [U]Short) - // - Mask widened product (SIMD32 [U]Short) to select relevant bits - // - Pack masked product so that relevant bits are packed together in upper and lower halves - // - Shuffle packed product so that relevant bits are placed together in the lower half - // - Select lower (SIMD16 [U]Byte) from shuffled product (SIMD32 [U]Short) - widenedSimdBaseJitType = simdBaseType == TYP_BYTE ? CORINFO_TYPE_SHORT : CORINFO_TYPE_USHORT; - widenIntrinsic = NI_AVX2_ConvertToVector256Int16; - widenedType = TYP_SIMD32; - widenedSimdSize = 32; - - // Vector256 widenedOp1 = Avx2.ConvertToVector256Int16(op1).AsUInt16() - GenTree* widenedOp1 = - gtNewSimdHWIntrinsicNode(widenedType, op1, widenIntrinsic, simdBaseJitType, simdSize); - - // Vector256 widenedOp2 = Avx2.ConvertToVector256Int16(op2).AsUInt16() - GenTree* widenedOp2 = - gtNewSimdHWIntrinsicNode(widenedType, op2, widenIntrinsic, simdBaseJitType, simdSize); - - // Vector256 widenedProduct = widenedOp1 * widenedOp2 - GenTree* widenedProduct = gtNewSimdBinOpNode(GT_MUL, widenedType, widenedOp1, widenedOp2, - widenedSimdBaseJitType, widenedSimdSize); - - // Vector256 vecCon1 = Vector256.Create(0x00FF00FF00FF00FF).AsUInt16() - GenTreeVecCon* vecCon1 = gtNewVconNode(widenedType); - - for (unsigned i = 0; i < (widenedSimdSize / 8); i++) - { - vecCon1->gtSimdVal.u64[i] = 0x00FF00FF00FF00FF; - } - - // Vector256 maskedProduct = Avx2.And(widenedProduct, vecCon1).AsInt16() - GenTree* maskedProduct = gtNewSimdBinOpNode(GT_AND, widenedType, widenedProduct, vecCon1, - widenedSimdBaseJitType, widenedSimdSize); - GenTree* maskedProductDup = fgMakeMultiUse(&maskedProduct); - - // Vector256 packedProduct = Avx2.PackUnsignedSaturate(maskedProduct, - // maskedProduct).AsUInt64() - GenTree* packedProduct = - gtNewSimdHWIntrinsicNode(widenedType, maskedProduct, maskedProductDup, - NI_AVX2_PackUnsignedSaturate, CORINFO_TYPE_UBYTE, widenedSimdSize); - - CorInfoType permuteBaseJitType = - (simdBaseType == TYP_BYTE) ? CORINFO_TYPE_LONG : CORINFO_TYPE_ULONG; - - // Vector256 shuffledProduct = Avx2.Permute4x64(w1, 0xD8).AsByte() - GenTree* shuffledProduct = - gtNewSimdHWIntrinsicNode(widenedType, packedProduct, gtNewIconNode(SHUFFLE_WYZX), - NI_AVX2_Permute4x64, permuteBaseJitType, widenedSimdSize); - - // Vector128 product = shuffledProduct.getLower() - return gtNewSimdGetLowerNode(type, shuffledProduct, simdBaseJitType, widenedSimdSize); - } - } - - // No special handling could be performed, apply fallback logic: - // - Widen both inputs lower and upper halves as [U]Short (using helper method) - // - Multiply corrsponding widened input halves together as widened product halves - // - Narrow widened product halves as [U]Byte (using helper method) - widenedSimdBaseJitType = simdBaseType == TYP_BYTE ? CORINFO_TYPE_SHORT : CORINFO_TYPE_USHORT; - - // op1Dup = op1 - GenTree* op1Dup = fgMakeMultiUse(&op1); - - // op2Dup = op2 - GenTree* op2Dup = fgMakeMultiUse(&op2); - - // Vector256 lowerOp1 = Avx2.ConvertToVector256Int16(op1.GetLower()).AsUInt16() - GenTree* lowerOp1 = gtNewSimdWidenLowerNode(type, op1, simdBaseJitType, simdSize); - - // Vector256 lowerOp2 = Avx2.ConvertToVector256Int16(op2.GetLower()).AsUInt16() - GenTree* lowerOp2 = gtNewSimdWidenLowerNode(type, op2, simdBaseJitType, simdSize); - - // Vector256 lowerProduct = lowerOp1 * lowerOp2 - GenTree* lowerProduct = - gtNewSimdBinOpNode(GT_MUL, type, lowerOp1, lowerOp2, widenedSimdBaseJitType, simdSize); - - // Vector256 upperOp1 = Avx2.ConvertToVector256Int16(op1.GetUpper()).AsUInt16() - GenTree* upperOp1 = gtNewSimdWidenUpperNode(type, op1Dup, simdBaseJitType, simdSize); - - // Vector256 upperOp2 = Avx2.ConvertToVector256Int16(op2.GetUpper()).AsUInt16() - GenTree* upperOp2 = gtNewSimdWidenUpperNode(type, op2Dup, simdBaseJitType, simdSize); - - // Vector256 upperProduct = upperOp1 * upperOp2 - GenTree* upperProduct = - gtNewSimdBinOpNode(GT_MUL, type, upperOp1, upperOp2, widenedSimdBaseJitType, simdSize); - - // Narrow and merge halves using helper method - return gtNewSimdNarrowNode(type, lowerProduct, upperProduct, simdBaseJitType, simdSize); - } - else if (varTypeIsInt(simdBaseType)) - { - // op1Dup = op1 - GenTree* op1Dup = fgMakeMultiUse(&op1); - - // op2Dup = op2 - GenTree* op2Dup = fgMakeMultiUse(&op2); - - // op1Dup = Sse2.ShiftRightLogical128BitLane(op1Dup, 4) - op1Dup = gtNewSimdHWIntrinsicNode(type, op1Dup, gtNewIconNode(4, TYP_INT), - NI_SSE2_ShiftRightLogical128BitLane, simdBaseJitType, simdSize); - - // op2Dup = Sse2.ShiftRightLogical128BitLane(op2Dup, 4) - op2Dup = gtNewSimdHWIntrinsicNode(type, op2Dup, gtNewIconNode(4, TYP_INT), - NI_SSE2_ShiftRightLogical128BitLane, simdBaseJitType, simdSize); - - // op2Dup = Sse2.Multiply(op1Dup.AsUInt32(), op2Dup.AsUInt32()).AsInt32() - op2Dup = gtNewSimdHWIntrinsicNode(type, op1Dup, op2Dup, NI_SSE2_Multiply, CORINFO_TYPE_ULONG, simdSize); - - // op2Dup = Sse2.Shuffle(op2Dup, (0, 0, 2, 0)) - op2Dup = gtNewSimdHWIntrinsicNode(type, op2Dup, gtNewIconNode(SHUFFLE_XXZX, TYP_INT), NI_SSE2_Shuffle, - simdBaseJitType, simdSize); - - // op1 = Sse2.Multiply(op1.AsUInt32(), op2.AsUInt32()).AsInt32() - op1 = gtNewSimdHWIntrinsicNode(type, op1, op2, NI_SSE2_Multiply, CORINFO_TYPE_ULONG, simdSize); - - // op1 = Sse2.Shuffle(op1, (0, 0, 2, 0)) - op1 = gtNewSimdHWIntrinsicNode(type, op1, gtNewIconNode(SHUFFLE_XXZX, TYP_INT), NI_SSE2_Shuffle, - simdBaseJitType, simdSize); - - // op2 = op2Dup; - op2 = op2Dup; - - // result = Sse2.UnpackLow(op1, op2) - return gtNewSimdHWIntrinsicNode(type, op1, op2, NI_SSE2_UnpackLow, simdBaseJitType, simdSize); - } - else if (varTypeIsLong(simdBaseType)) - { - assert((simdSize == 16) || (simdSize == 32) || (simdSize == 64)); - - assert(((simdSize == 16) && compOpportunisticallyDependsOn(InstructionSet_SSE41)) || - ((simdSize == 32) && compOpportunisticallyDependsOn(InstructionSet_AVX2))); - - // Make op1 and op2 multi-use: - GenTree* op1Dup = fgMakeMultiUse(&op1); - GenTree* op2Dup = fgMakeMultiUse(&op2); - - const bool is256 = simdSize == 32; - - // Vector256 tmp0 = Avx2.Multiply(left, right); - GenTreeHWIntrinsic* tmp0 = - gtNewSimdHWIntrinsicNode(type, op1, op2, is256 ? NI_AVX2_Multiply : NI_SSE2_Multiply, - CORINFO_TYPE_ULONG, simdSize); - - // Vector256 tmp1 = Avx2.Shuffle(right.AsUInt32(), ZWXY); - GenTree* shuffleMask = gtNewIconNode(SHUFFLE_ZWXY, TYP_INT); - GenTreeHWIntrinsic* tmp1 = - gtNewSimdHWIntrinsicNode(type, op2Dup, shuffleMask, is256 ? NI_AVX2_Shuffle : NI_SSE2_Shuffle, - CORINFO_TYPE_UINT, simdSize); - - // Vector256 tmp2 = Avx2.MultiplyLow(left.AsUInt32(), tmp1); - GenTree* tmp2 = gtNewSimdBinOpNode(GT_MUL, type, op1Dup, tmp1, CORINFO_TYPE_UINT, simdSize); - - // Vector256 tmp3 = Avx2.HorizontalAdd(tmp2.AsInt32(), Vector256.Zero); - GenTreeHWIntrinsic* tmp3 = - gtNewSimdHWIntrinsicNode(type, tmp2, gtNewZeroConNode(type), - is256 ? NI_AVX2_HorizontalAdd : NI_SSSE3_HorizontalAdd, CORINFO_TYPE_UINT, - simdSize); - - // Vector256 tmp4 = Avx2.Shuffle(tmp3, YWXW); - shuffleMask = gtNewIconNode(SHUFFLE_YWXW, TYP_INT); - GenTreeHWIntrinsic* tmp4 = - gtNewSimdHWIntrinsicNode(type, tmp3, shuffleMask, is256 ? NI_AVX2_Shuffle : NI_SSE2_Shuffle, - CORINFO_TYPE_UINT, simdSize); - - // result = tmp0 + tmp4; - return gtNewSimdBinOpNode(GT_ADD, type, tmp0, tmp4, simdBaseJitType, simdSize); - } -#elif defined(TARGET_ARM64) - if (varTypeIsLong(simdBaseType)) - { - GenTree** op2ToDup = nullptr; - - assert(varTypeIsSIMD(op1)); - op1 = gtNewSimdToScalarNode(TYP_LONG, op1, simdBaseJitType, simdSize); - GenTree** op1ToDup = &op1->AsHWIntrinsic()->Op(1); - - if (varTypeIsSIMD(op2)) - { - op2 = gtNewSimdToScalarNode(TYP_LONG, op2, simdBaseJitType, simdSize); - op2ToDup = &op2->AsHWIntrinsic()->Op(1); - } - - // lower = op1.GetElement(0) * op2.GetElement(0) - GenTree* lower = gtNewOperNode(GT_MUL, TYP_LONG, op1, op2); - - if (op2ToDup == nullptr) - { - op2ToDup = &lower->AsOp()->gtOp2; - } - lower = gtNewSimdCreateScalarUnsafeNode(type, lower, simdBaseJitType, simdSize); - - if (simdSize == 8) - { - // return Vector64.CreateScalarUnsafe(lower) - return lower; - } - - // Make the original op1 and op2 multi-use: - GenTree* op1Dup = fgMakeMultiUse(op1ToDup); - GenTree* op2Dup = fgMakeMultiUse(op2ToDup); - - assert(!varTypeIsArithmetic(op1Dup)); - op1Dup = gtNewSimdGetElementNode(TYP_LONG, op1Dup, gtNewIconNode(1), simdBaseJitType, simdSize); - - if (!varTypeIsArithmetic(op2Dup)) - { - op2Dup = gtNewSimdGetElementNode(TYP_LONG, op2Dup, gtNewIconNode(1), simdBaseJitType, simdSize); - } - - // upper = op1.GetElement(1) * op2.GetElement(1) - GenTree* upper = gtNewOperNode(GT_MUL, TYP_LONG, op1Dup, op2Dup); - - // return Vector128.Create(lower, upper) - return gtNewSimdWithElementNode(type, lower, gtNewIconNode(1), upper, simdBaseJitType, simdSize); - } -#endif // !TARGET_XARCH && !TARGET_ARM64 - unreached(); - } - - default: - { - unreached(); - } - } -} - -GenTree* Compiler::gtNewSimdCeilNode(var_types type, GenTree* op1, CorInfoType simdBaseJitType, unsigned simdSize) -{ - assert(IsBaselineSimdIsaSupportedDebugOnly()); - - assert(varTypeIsSIMD(type)); - assert(getSIMDTypeForSize(simdSize) == type); - - assert(op1 != nullptr); - assert(op1->TypeIs(type)); - - var_types simdBaseType = JitType2PreciseVarType(simdBaseJitType); - assert(varTypeIsFloating(simdBaseType)); - - NamedIntrinsic intrinsic = NI_Illegal; - -#if defined(TARGET_XARCH) - if (simdSize == 32) - { - assert(compIsaSupportedDebugOnly(InstructionSet_AVX)); - intrinsic = NI_AVX_Ceiling; - } - else if (simdSize == 64) - { - assert(compIsaSupportedDebugOnly(InstructionSet_AVX512F)); - GenTree* op2 = gtNewIconNode(static_cast(FloatRoundingMode::ToPositiveInfinity)); - return gtNewSimdHWIntrinsicNode(type, op1, op2, NI_AVX512F_RoundScale, simdBaseJitType, simdSize); - } - else - { - assert(compIsaSupportedDebugOnly(InstructionSet_SSE41)); - intrinsic = NI_SSE41_Ceiling; - } -#elif defined(TARGET_ARM64) - if (simdBaseType == TYP_DOUBLE) - { - intrinsic = (simdSize == 8) ? NI_AdvSimd_CeilingScalar : NI_AdvSimd_Arm64_Ceiling; - } - else - { - intrinsic = NI_AdvSimd_Ceiling; - } -#else -#error Unsupported platform -#endif // !TARGET_XARCH && !TARGET_ARM64 - - assert(intrinsic != NI_Illegal); - return gtNewSimdHWIntrinsicNode(type, op1, intrinsic, simdBaseJitType, simdSize); -} - -#if defined(FEATURE_MASKED_HW_INTRINSICS) -//------------------------------------------------------------------------ -// gtNewSimdCvtMaskToVectorNode: Convert a HW instrinsic mask node to a vector -// -// Arguments: -// type -- The type of the node to convert to -// op1 -- The node to convert -// simdBaseJitType -- the base jit type of the converted node -// simdSize -- the simd size of the converted node -// -// Return Value: -// The node converted to the given type -// -GenTree* Compiler::gtNewSimdCvtMaskToVectorNode(var_types type, - GenTree* op1, - CorInfoType simdBaseJitType, - unsigned simdSize) -{ - assert(varTypeIsMask(op1)); - assert(varTypeIsSIMD(type)); - compMaskConvertUsed = true; - -#if defined(TARGET_XARCH) - return gtNewSimdHWIntrinsicNode(type, op1, NI_EVEX_ConvertMaskToVector, simdBaseJitType, simdSize); -#elif defined(TARGET_ARM64) - return gtNewSimdHWIntrinsicNode(type, op1, NI_Sve_ConvertMaskToVector, simdBaseJitType, simdSize); -#else -#error Unsupported platform -#endif // !TARGET_XARCH && !TARGET_ARM64 -} -#endif // FEATURE_MASKED_HW_INTRINSICS - -GenTree* Compiler::gtNewSimdCvtNode(var_types type, - GenTree* op1, - CorInfoType simdTargetBaseJitType, - CorInfoType simdSourceBaseJitType, - unsigned simdSize) -{ - assert(varTypeIsSIMD(type)); - assert(getSIMDTypeForSize(simdSize) == type); - assert(op1 != nullptr); - assert(op1->TypeIs(type)); - - var_types simdSourceBaseType = JitType2PreciseVarType(simdSourceBaseJitType); - var_types simdTargetBaseType = JitType2PreciseVarType(simdTargetBaseJitType); - assert(varTypeIsFloating(simdSourceBaseType)); - assert(varTypeIsIntegral(simdTargetBaseType)); - - assert(IsBaselineSimdIsaSupportedDebugOnly()); - -#if defined(TARGET_XARCH) - assert(IsBaselineVector512IsaSupportedDebugOnly() || - (simdSize != 64 && compIsaSupportedDebugOnly(InstructionSet_AVX10v1)) || - ((simdTargetBaseType == TYP_INT) && ((simdSize == 16 && compIsaSupportedDebugOnly(InstructionSet_SSE41)) || - (simdSize == 32 && compIsaSupportedDebugOnly(InstructionSet_AVX))))); - - GenTree* fixupVal; - bool isV512Supported = false; - - if (compIsEvexOpportunisticallySupported(isV512Supported)) - { - /*Generate the control table for VFIXUPIMMSD/SS - - For conversion to unsigned - // QNAN: 0b1000: Saturate to Zero - // SNAN: 0b1000: Saturate to Zero - // ZERO: 0b0000 - // +ONE: 0b0000 - // -INF: 0b1000: Saturate to Zero - // +INF: 0b0000 - // -VAL: 0b1000: Saturate to Zero - // +VAL: 0b0000 - - For conversion to signed - // QNAN: 0b1000: Saturate to Zero - // SNAN: 0b1000: Saturate to Zero - // ZERO: 0b0000 - // +ONE: 0b0000 - // -INF: 0b0000 - // +INF: 0b0000 - // -VAL: 0b0000 - // +VAL: 0b0000 - */ - int32_t iconVal = varTypeIsUnsigned(simdTargetBaseType) ? 0x08080088 : 0x00000088; - GenTree* tblCon = gtNewSimdCreateBroadcastNode(type, gtNewIconNode(iconVal), simdTargetBaseJitType, simdSize); - - // We need op1Clone to run fixup - GenTree* op1Clone = fgMakeMultiUse(&op1); - NamedIntrinsic fixupHwIntrinsicID; - - if (simdSize == 64) - { - fixupHwIntrinsicID = NI_AVX512F_Fixup; - } - else - { - fixupHwIntrinsicID = !isV512Supported ? NI_AVX10v1_Fixup : NI_AVX512F_VL_Fixup; - } - // run vfixupimmsd base on table and no flags reporting - fixupVal = gtNewSimdHWIntrinsicNode(type, op1, op1Clone, tblCon, gtNewIconNode(0), fixupHwIntrinsicID, - simdSourceBaseJitType, simdSize); - } - else - { - // Zero out NaN values from the input. - // mask1 contains the output either 0xFFFFFFFF or 0. - // FixupVal zeros out any NaN values in the input by ANDing input with mask1. - GenTree* op1Clone1 = fgMakeMultiUse(&op1); - GenTree* mask1 = gtNewSimdIsNaNNode(type, op1, simdSourceBaseJitType, simdSize); - fixupVal = gtNewSimdBinOpNode(GT_AND_NOT, type, op1Clone1, mask1, simdSourceBaseJitType, simdSize); - } - - if (varTypeIsSigned(simdTargetBaseType)) - { - GenTree* maxVal; - GenTree* maxValDup; - if (varTypeIsLong(simdTargetBaseType)) - { - int64_t actualMaxVal = INT64_MAX; - maxVal = gtNewDconNode(static_cast(actualMaxVal), simdSourceBaseType); - maxVal = gtNewSimdCreateBroadcastNode(type, maxVal, simdSourceBaseJitType, simdSize); - maxValDup = - gtNewSimdCreateBroadcastNode(type, gtNewLconNode(actualMaxVal), simdTargetBaseJitType, simdSize); - } - else - { - ssize_t actualMaxVal = INT32_MAX; - maxVal = gtNewDconNode(static_cast(actualMaxVal), simdSourceBaseType); - maxVal = gtNewSimdCreateBroadcastNode(type, maxVal, simdSourceBaseJitType, simdSize); - maxValDup = - gtNewSimdCreateBroadcastNode(type, gtNewIconNode(actualMaxVal), simdTargetBaseJitType, simdSize); - } - - // we will be using the input value twice - GenTree* fixupValDup = fgMakeMultiUse(&fixupVal); - - // compare with max value of integer/long - fixupVal = gtNewSimdCmpOpNode(GT_GE, type, fixupVal, maxVal, simdSourceBaseJitType, simdSize); - - // cast it - GenTree* castNode = - gtNewSimdCvtNativeNode(type, fixupValDup, simdTargetBaseJitType, simdSourceBaseJitType, simdSize); - - // use the fixupVal mask with input value and max value to blend - return gtNewSimdCndSelNode(type, fixupVal, maxValDup, castNode, simdTargetBaseJitType, simdSize); - } - else - { - return gtNewSimdCvtNativeNode(type, fixupVal, simdTargetBaseJitType, simdSourceBaseJitType, simdSize); - } -#elif defined(TARGET_ARM64) - return gtNewSimdCvtNativeNode(type, op1, simdTargetBaseJitType, simdSourceBaseJitType, simdSize); -#else -#error Unsupported platform -#endif // !TARGET_XARCH && !TARGET_ARM64 -} - -GenTree* Compiler::gtNewSimdCvtNativeNode(var_types type, - GenTree* op1, - CorInfoType simdTargetBaseJitType, - CorInfoType simdSourceBaseJitType, - unsigned simdSize) -{ - assert(varTypeIsSIMD(type)); - assert(getSIMDTypeForSize(simdSize) == type); - assert(op1 != nullptr); - assert(op1->TypeIs(type)); - - var_types simdSourceBaseType = JitType2PreciseVarType(simdSourceBaseJitType); - var_types simdTargetBaseType = JitType2PreciseVarType(simdTargetBaseJitType); - assert(varTypeIsFloating(simdSourceBaseType)); - assert(varTypeIsIntegral(simdTargetBaseType)); - - assert(IsBaselineSimdIsaSupportedDebugOnly()); - - // Generate intrinsic needed for conversion - NamedIntrinsic hwIntrinsicID = NI_Illegal; - -#if defined(TARGET_XARCH) - assert(IsBaselineVector512IsaSupportedDebugOnly() || - (simdSize != 64 && compIsaSupportedDebugOnly(InstructionSet_AVX10v1)) || - ((simdTargetBaseType == TYP_INT) && - ((simdSize == 16) || (simdSize == 32 && compIsaSupportedDebugOnly(InstructionSet_AVX))))); - - switch (simdSourceBaseJitType) - { - case CORINFO_TYPE_FLOAT: - { - switch (simdTargetBaseJitType) - { - case CORINFO_TYPE_INT: - { - switch (simdSize) - { - case 64: - { - hwIntrinsicID = NI_AVX512F_ConvertToVector512Int32WithTruncation; - break; - } - - case 32: - { - hwIntrinsicID = NI_AVX_ConvertToVector256Int32WithTruncation; - break; - } - - case 16: - { - hwIntrinsicID = NI_SSE2_ConvertToVector128Int32WithTruncation; - break; - } - - default: - unreached(); - } - break; - } - - case CORINFO_TYPE_UINT: - { - switch (simdSize) - { - case 64: - { - hwIntrinsicID = NI_AVX512F_ConvertToVector512UInt32WithTruncation; - break; - } - - case 32: - { - hwIntrinsicID = compOpportunisticallyDependsOn(InstructionSet_AVX10v1) - ? NI_AVX10v1_ConvertToVector256UInt32WithTruncation - : NI_AVX512F_VL_ConvertToVector256UInt32WithTruncation; - break; - } - - case 16: - { - hwIntrinsicID = compOpportunisticallyDependsOn(InstructionSet_AVX10v1) - ? NI_AVX10v1_ConvertToVector128UInt32WithTruncation - : NI_AVX512F_VL_ConvertToVector128UInt32WithTruncation; - break; - } - - default: - unreached(); - } - break; - } - - default: - unreached(); - } - break; - } - - case CORINFO_TYPE_DOUBLE: - { - switch (simdTargetBaseJitType) - { - case CORINFO_TYPE_LONG: - { - switch (simdSize) - { - case 64: - { - hwIntrinsicID = NI_AVX512DQ_ConvertToVector512Int64WithTruncation; - break; - } - - case 32: - { - hwIntrinsicID = compOpportunisticallyDependsOn(InstructionSet_AVX10v1) - ? NI_AVX10v1_ConvertToVector256Int64WithTruncation - : NI_AVX512DQ_VL_ConvertToVector256Int64WithTruncation; - break; - } - - case 16: - { - hwIntrinsicID = compOpportunisticallyDependsOn(InstructionSet_AVX10v1) - ? NI_AVX10v1_ConvertToVector128Int64WithTruncation - : NI_AVX512DQ_VL_ConvertToVector128Int64WithTruncation; - break; - } - - default: - unreached(); - } - break; - } - - case CORINFO_TYPE_ULONG: - { - switch (simdSize) - { - case 64: - { - hwIntrinsicID = NI_AVX512DQ_ConvertToVector512UInt64WithTruncation; - break; - } - - case 32: - { - hwIntrinsicID = compOpportunisticallyDependsOn(InstructionSet_AVX10v1) - ? NI_AVX10v1_ConvertToVector256UInt64WithTruncation - : NI_AVX512DQ_VL_ConvertToVector256UInt64WithTruncation; - break; - } - - case 16: - { - hwIntrinsicID = compOpportunisticallyDependsOn(InstructionSet_AVX10v1) - ? NI_AVX10v1_ConvertToVector128UInt64WithTruncation - : NI_AVX512DQ_VL_ConvertToVector128UInt64WithTruncation; - break; - } - - default: - unreached(); - } - break; - } - - default: - unreached(); - } - break; - } - - default: - unreached(); - } -#elif defined(TARGET_ARM64) - assert((simdSize == 8) || (simdSize == 16)); - - switch (simdSourceBaseJitType) - { - case CORINFO_TYPE_FLOAT: - { - switch (simdTargetBaseJitType) - { - case CORINFO_TYPE_INT: - { - hwIntrinsicID = NI_AdvSimd_ConvertToInt32RoundToZero; - break; - } - - case CORINFO_TYPE_UINT: - { - hwIntrinsicID = NI_AdvSimd_ConvertToUInt32RoundToZero; - break; - } - - default: - unreached(); - } - break; - } - - case CORINFO_TYPE_DOUBLE: - { - switch (simdTargetBaseJitType) - { - case CORINFO_TYPE_LONG: - { - hwIntrinsicID = (simdSize == 8) ? NI_AdvSimd_Arm64_ConvertToInt64RoundToZeroScalar - : NI_AdvSimd_Arm64_ConvertToInt64RoundToZero; - break; - } - - case CORINFO_TYPE_ULONG: - { - hwIntrinsicID = (simdSize == 8) ? NI_AdvSimd_Arm64_ConvertToUInt64RoundToZeroScalar - : NI_AdvSimd_Arm64_ConvertToUInt64RoundToZero; - break; - } - - default: - unreached(); - } - break; - } - - default: - unreached(); - } -#else -#error Unsupported platform -#endif // !TARGET_XARCH && !TARGET_ARM64 - - assert(hwIntrinsicID != NI_Illegal); - return gtNewSimdHWIntrinsicNode(type, op1, hwIntrinsicID, simdSourceBaseJitType, simdSize); -} - -#if defined(FEATURE_MASKED_HW_INTRINSICS) -//------------------------------------------------------------------------ -// gtNewSimdCvtVectorToMaskNode: Convert a HW instrinsic vector node to a mask -// -// Arguments: -// type -- The type of the mask to produce. -// op1 -- The node to convert -// simdBaseJitType -- the base jit type of the converted node -// simdSize -- the simd size of the converted node -// -// Return Value: -// The node converted to the a mask type -// -GenTree* Compiler::gtNewSimdCvtVectorToMaskNode(var_types type, - GenTree* op1, - CorInfoType simdBaseJitType, - unsigned simdSize) -{ - assert(varTypeIsMask(type)); - assert(varTypeIsSIMD(op1)); - compMaskConvertUsed = true; - -#if defined(TARGET_XARCH) - return gtNewSimdHWIntrinsicNode(TYP_MASK, op1, NI_EVEX_ConvertVectorToMask, simdBaseJitType, simdSize); -#elif defined(TARGET_ARM64) - // We use cmpne which requires an embedded mask. - GenTree* trueMask = gtNewSimdAllTrueMaskNode(simdBaseJitType, simdSize); - return gtNewSimdHWIntrinsicNode(TYP_MASK, trueMask, op1, NI_Sve_ConvertVectorToMask, simdBaseJitType, simdSize); -#else -#error Unsupported platform -#endif // !TARGET_XARCH && !TARGET_ARM64 -} -#endif // FEATURE_MASKED_HW_INTRINSICS - -GenTree* Compiler::gtNewSimdCmpOpNode( - genTreeOps op, var_types type, GenTree* op1, GenTree* op2, CorInfoType simdBaseJitType, unsigned simdSize) -{ - assert(IsBaselineSimdIsaSupportedDebugOnly()); - - assert(varTypeIsSIMD(type)); - assert(getSIMDTypeForSize(simdSize) == type); - - assert(op1 != nullptr); - assert(op1->TypeIs(type)); - - assert(op2 != nullptr); - assert(op2->TypeIs(type)); - - var_types simdBaseType = JitType2PreciseVarType(simdBaseJitType); - assert(varTypeIsArithmetic(simdBaseType)); - - var_types lookupType = GenTreeHWIntrinsic::GetLookupTypeForCmpOp(this, op, type, simdBaseType, simdSize); - - NamedIntrinsic intrinsic = - GenTreeHWIntrinsic::GetHWIntrinsicIdForCmpOp(this, op, lookupType, op1, op2, simdBaseType, simdSize, false); - - if (intrinsic != NI_Illegal) - { -#if defined(FEATURE_MASKED_HW_INTRINSICS) - if (lookupType != type) - { - assert(varTypeIsMask(lookupType)); - GenTree* retNode = gtNewSimdHWIntrinsicNode(lookupType, op1, op2, intrinsic, simdBaseJitType, simdSize); - return gtNewSimdCvtMaskToVectorNode(type, retNode, simdBaseJitType, simdSize); - } -#else - assert(lookupType == type); -#endif // !FEATURE_MASKED_HW_INTRINSICS - return gtNewSimdHWIntrinsicNode(type, op1, op2, intrinsic, simdBaseJitType, simdSize); - } - - assert(lookupType == type); - - switch (op) - { -#if defined(TARGET_XARCH) - case GT_EQ: - { - assert(varTypeIsLong(simdBaseType)); - assert(simdSize == 16); - - // There is no direct SSE2 support for comparing TYP_LONG vectors. - // These have to be implemented in terms of TYP_INT vector comparison operations. - // - // tmp = (op1 == op2) i.e. compare for equality as if op1 and op2 are vector of int - // op1 = tmp - // op2 = Shuffle(op1, (2, 3, 0, 1)) - // result = BitwiseAnd(tmp, op2) - // - // Shuffle is meant to swap the comparison results of low-32-bits and high 32-bits of - // respective long elements. - - GenTree* tmp = gtNewSimdCmpOpNode(op, type, op1, op2, CORINFO_TYPE_INT, simdSize); - - op1 = fgMakeMultiUse(&tmp); - op2 = gtNewSimdHWIntrinsicNode(type, op1, gtNewIconNode(SHUFFLE_ZWXY), NI_SSE2_Shuffle, CORINFO_TYPE_INT, - simdSize); - - return gtNewSimdBinOpNode(GT_AND, type, tmp, op2, simdBaseJitType, simdSize); - } - - case GT_GE: - case GT_LE: - { - assert(varTypeIsIntegral(simdBaseType)); - assert(!canUseEvexEncodingDebugOnly()); - - // If we don't have an intrinsic set for this, try "Max(op1, op2) == op1" for GE - // and "Min(op1, op2) == op1" for LE - // - // NOTE: technically, we can special case byte type to only require SSE2, but it - // complicates the test matrix for little gains. - - if (((simdSize == 32) && compOpportunisticallyDependsOn(InstructionSet_AVX2)) || - ((simdSize == 16) && compOpportunisticallyDependsOn(InstructionSet_SSE41))) - { - // TODO-AVX512: We can use this trick for longs only with AVX-512 - if (!varTypeIsLong(simdBaseType)) - { - assert(!varTypeIsFloating(simdBaseType)); - GenTree* op1Dup = fgMakeMultiUse(&op1); - - if (op == GT_GE) - { - // EQ(Max(op1, op2), op1) - op1 = gtNewSimdMaxNativeNode(type, op1, op2, simdBaseJitType, simdSize); - } - else - { - // EQ(Min(op1, op2), op1) - op1 = gtNewSimdMinNativeNode(type, op1, op2, simdBaseJitType, simdSize); - } - return gtNewSimdCmpOpNode(GT_EQ, type, op1, op1Dup, simdBaseJitType, simdSize); - } - } - - // There is no direct support for doing a combined comparison and equality for integral types. - // These have to be implemented by performing both halves and combining their results. - // - // op1Dup = op1 - // op2Dup = op2 - // - // For greater than: - // op1 = GreaterThan(op1, op2) - // op2 = Equals(op1Dup, op2Dup) - // - // For less than: - // op1 = LessThan(op1, op2) - // op2 = Equals(op1Dup, op2Dup) - // - // result = BitwiseOr(op1, op2) - - GenTree* op1Dup = fgMakeMultiUse(&op1); - GenTree* op2Dup = fgMakeMultiUse(&op2); - - if (op == GT_GE) - { - op = GT_GT; - } - else - { - op = GT_LT; - } - - op1 = gtNewSimdCmpOpNode(op, type, op1, op2, simdBaseJitType, simdSize); - op2 = gtNewSimdCmpOpNode(GT_EQ, type, op1Dup, op2Dup, simdBaseJitType, simdSize); - - return gtNewSimdBinOpNode(GT_OR, type, op1, op2, simdBaseJitType, simdSize); - } - - case GT_GT: - case GT_LT: - { - if (varTypeIsUnsigned(simdBaseType)) - { - // Vector of byte, ushort, uint and ulong: - // Hardware supports > and < for signed comparison. Therefore, to use it for - // comparing unsigned numbers, we subtract a constant from both the - // operands such that the result fits within the corresponding signed - // type. The resulting signed numbers are compared using signed comparison. - // - // Vector of byte: constant to be subtracted is 2^7 - // Vector of ushort: constant to be subtracted is 2^15 - // Vector of uint: constant to be subtracted is 2^31 - // Vector of ulong: constant to be subtracted is 2^63 - // - // We need to treat op1 and op2 as signed for comparison purpose after - // the transformation. - - uint64_t constVal = 0; - CorInfoType opJitType = simdBaseJitType; - var_types opType = simdBaseType; - - switch (simdBaseType) - { - case TYP_UBYTE: - { - constVal = 0x8080808080808080; - simdBaseJitType = CORINFO_TYPE_BYTE; - simdBaseType = TYP_BYTE; - break; - } - - case TYP_USHORT: - { - constVal = 0x8000800080008000; - simdBaseJitType = CORINFO_TYPE_SHORT; - simdBaseType = TYP_SHORT; - break; - } - - case TYP_UINT: - { - constVal = 0x8000000080000000; - simdBaseJitType = CORINFO_TYPE_INT; - simdBaseType = TYP_INT; - break; - } - - case TYP_ULONG: - { - constVal = 0x8000000000000000; - simdBaseJitType = CORINFO_TYPE_LONG; - simdBaseType = TYP_LONG; - break; - } - - default: - { - unreached(); - } - } - - GenTreeVecCon* vecCon1 = gtNewVconNode(type); - - for (unsigned i = 0; i < (simdSize / 8); i++) - { - vecCon1->gtSimdVal.u64[i] = constVal; - } - - GenTree* vecCon2 = gtCloneExpr(vecCon1); - - // op1 = op1 - constVector - op1 = gtNewSimdBinOpNode(GT_SUB, type, op1, vecCon1, opJitType, simdSize); - - // op2 = op2 - constVector - op2 = gtNewSimdBinOpNode(GT_SUB, type, op2, vecCon2, opJitType, simdSize); - - return gtNewSimdCmpOpNode(op, type, op1, op2, simdBaseJitType, simdSize); - } - else - { - assert(varTypeIsLong(simdBaseType)); - assert(simdSize == 16); - - // There is no direct SSE2 support for comparing TYP_LONG vectors. - // These have to be implemented in terms of TYP_INT vector comparison operations. - // - // Let us consider the case of single long element comparison. - // Say op1 = (x1, y1) and op2 = (x2, y2) where x1, y1, x2, and y2 are 32-bit integers that comprise - // the - // longs op1 and op2. - // - // GreaterThan(op1, op2) can be expressed in terms of > relationship between 32-bit integers that - // comprise op1 and op2 as - // = (x1, y1) > (x2, y2) - // = (x1 > x2) || [(x1 == x2) && (y1 > y2)] - eq (1) - // - // LessThan(op1, op2) can be expressed in terms of < relationship between 32-bit integers that - // comprise op1 and op2 as - // = (x1, y1) < (x2, y2) - // = (x1 < x2) || [(x1 == x2) && (y1 < y2)] - eq (1) - // - // op1Dup1 = op1 - // op1Dup2 = op1Dup1 - // op2Dup1 = op2 - // op2Dup2 = op2Dup1 - // - // t = (op1 cmp op2) - 32-bit signed comparison - // u = (op1Dup1 == op2Dup1) - 32-bit equality comparison - // v = (op1Dup2 cmp op2Dup2) - 32-bit unsigned comparison - // - // op1 = Shuffle(t, (3, 3, 1, 1)) - This corresponds to (x1 > x2) in eq(1) above - // u = Shuffle(u, (3, 3, 1, 1)) - This corresponds to (x1 == x2) in eq(1) above - // v = Shuffle(v, (2, 2, 0, 0)) - This corresponds to (y1 > y2) in eq(1) above - // op2 = BitwiseAnd(u, v) - This corresponds to [(x1 == x2) && (y1 > y2)] in eq(1) above - // - // result = BitwiseOr(op1, op2) - - GenTree* op1Dup1 = fgMakeMultiUse(&op1); - GenTree* op1Dup2 = gtCloneExpr(op1Dup1); - - GenTree* op2Dup1 = fgMakeMultiUse(&op2); - GenTree* op2Dup2 = gtCloneExpr(op2Dup1); - - GenTree* t = gtNewSimdCmpOpNode(op, type, op1, op2, CORINFO_TYPE_INT, simdSize); - GenTree* u = gtNewSimdCmpOpNode(GT_EQ, type, op1Dup1, op2Dup1, CORINFO_TYPE_INT, simdSize); - GenTree* v = gtNewSimdCmpOpNode(op, type, op1Dup2, op2Dup2, CORINFO_TYPE_UINT, simdSize); - - op1 = gtNewSimdHWIntrinsicNode(type, t, gtNewIconNode(SHUFFLE_WWYY, TYP_INT), NI_SSE2_Shuffle, - CORINFO_TYPE_INT, simdSize); - u = gtNewSimdHWIntrinsicNode(type, u, gtNewIconNode(SHUFFLE_WWYY, TYP_INT), NI_SSE2_Shuffle, - CORINFO_TYPE_INT, simdSize); - v = gtNewSimdHWIntrinsicNode(type, v, gtNewIconNode(SHUFFLE_ZZXX, TYP_INT), NI_SSE2_Shuffle, - CORINFO_TYPE_INT, simdSize); - - op2 = gtNewSimdBinOpNode(GT_AND, type, u, v, simdBaseJitType, simdSize); - return gtNewSimdBinOpNode(GT_OR, type, op1, op2, simdBaseJitType, simdSize); - } - break; - } -#endif // TARGET_XARCH - - case GT_NE: - { -#if defined(TARGET_XARCH) - assert(varTypeIsIntegral(simdBaseType)); - assert(!canUseEvexEncodingDebugOnly()); -#endif // TARGET_XARCH - - GenTree* result = gtNewSimdCmpOpNode(GT_EQ, type, op1, op2, simdBaseJitType, simdSize); - return gtNewSimdUnOpNode(GT_NOT, type, result, simdBaseJitType, simdSize); - } - - default: - { - unreached(); - } - } -} - -GenTree* Compiler::gtNewSimdCmpOpAllNode( - genTreeOps op, var_types type, GenTree* op1, GenTree* op2, CorInfoType simdBaseJitType, unsigned simdSize) -{ - assert(IsBaselineSimdIsaSupportedDebugOnly()); - assert(type == TYP_INT); - - var_types simdType = getSIMDTypeForSize(simdSize); - assert(varTypeIsSIMD(simdType)); - - assert(op1 != nullptr); - assert(op1->TypeIs(simdType)); - - assert(op2 != nullptr); - assert(op2->TypeIs(simdType)); - - var_types simdBaseType = JitType2PreciseVarType(simdBaseJitType); - assert(varTypeIsArithmetic(simdBaseType)); - - NamedIntrinsic intrinsic = NI_Illegal; - - switch (op) - { -#if defined(TARGET_XARCH) - case GT_EQ: - { - if (simdSize == 32) - { - assert(compIsaSupportedDebugOnly(InstructionSet_AVX)); - assert(varTypeIsFloating(simdBaseType) || compIsaSupportedDebugOnly(InstructionSet_AVX2)); - - intrinsic = NI_Vector256_op_Equality; - } - else if (simdSize == 64) - { - assert(IsBaselineVector512IsaSupportedDebugOnly()); - intrinsic = NI_Vector512_op_Equality; - } - else - { - intrinsic = NI_Vector128_op_Equality; - } - break; - } - - case GT_GE: - case GT_GT: - case GT_LE: - case GT_LT: - { - // We want to generate a comparison along the lines of - // GT_XX(op1, op2).As() == Vector128.AllBitsSet - - if (simdSize == 32) - { - // TODO-XArch-CQ: It's a non-trivial amount of work to support these - // for floating-point while only utilizing AVX. It would require, among - // other things, inverting the comparison and potentially support for a - // new Avx.TestNotZ intrinsic to ensure the codegen remains efficient. - assert(compIsaSupportedDebugOnly(InstructionSet_AVX2)); - intrinsic = NI_Vector256_op_Equality; - } - else if (simdSize == 64) - { - assert(IsBaselineVector512IsaSupportedDebugOnly()); - intrinsic = NI_Vector512_op_Equality; - } - else - { - intrinsic = NI_Vector128_op_Equality; - } - - op1 = gtNewSimdCmpOpNode(op, simdType, op1, op2, simdBaseJitType, simdSize); - op2 = gtNewAllBitsSetConNode(simdType); - - if (simdBaseType == TYP_FLOAT) - { - simdBaseType = TYP_INT; - simdBaseJitType = CORINFO_TYPE_INT; - } - else if (simdBaseType == TYP_DOUBLE) - { - simdBaseType = TYP_LONG; - simdBaseJitType = CORINFO_TYPE_LONG; - } - break; - } -#elif defined(TARGET_ARM64) - case GT_EQ: - { - intrinsic = (simdSize == 8) ? NI_Vector64_op_Equality : NI_Vector128_op_Equality; - break; - } - - case GT_GE: - case GT_GT: - case GT_LE: - case GT_LT: - { - // We want to generate a comparison along the lines of - // GT_XX(op1, op2).As() == Vector128.AllBitsSet - - if (simdSize == 8) - { - intrinsic = NI_Vector64_op_Equality; - } - else - { - intrinsic = NI_Vector128_op_Equality; - } - - op1 = gtNewSimdCmpOpNode(op, simdType, op1, op2, simdBaseJitType, simdSize); - op2 = gtNewAllBitsSetConNode(simdType); - - if (simdBaseType == TYP_FLOAT) - { - simdBaseType = TYP_INT; - simdBaseJitType = CORINFO_TYPE_INT; - } - else if (simdBaseType == TYP_DOUBLE) - { - simdBaseType = TYP_LONG; - simdBaseJitType = CORINFO_TYPE_LONG; - } - break; - } -#else -#error Unsupported platform -#endif // !TARGET_XARCH && !TARGET_ARM64 - - default: - { - unreached(); - } - } - - assert(intrinsic != NI_Illegal); - return gtNewSimdHWIntrinsicNode(type, op1, op2, intrinsic, simdBaseJitType, simdSize); -} - -GenTree* Compiler::gtNewSimdCmpOpAnyNode( - genTreeOps op, var_types type, GenTree* op1, GenTree* op2, CorInfoType simdBaseJitType, unsigned simdSize) -{ - assert(IsBaselineSimdIsaSupportedDebugOnly()); - assert(type == TYP_INT); - - var_types simdType = getSIMDTypeForSize(simdSize); - assert(varTypeIsSIMD(simdType)); - - assert(op1 != nullptr); - assert(op1->TypeIs(simdType)); - - assert(op2 != nullptr); - assert(op2->TypeIs(simdType)); - - var_types simdBaseType = JitType2PreciseVarType(simdBaseJitType); - assert(varTypeIsArithmetic(simdBaseType)); - - NamedIntrinsic intrinsic = NI_Illegal; - - switch (op) - { -#if defined(TARGET_XARCH) - case GT_EQ: - case GT_GE: - case GT_GT: - case GT_LE: - case GT_LT: - { - // We want to generate a comparison along the lines of - // GT_XX(op1, op2).As() != Vector128.Zero - - if (simdSize == 32) - { - // TODO-XArch-CQ: It's a non-trivial amount of work to support these - // for floating-point while only utilizing AVX. It would require, among - // other things, inverting the comparison and potentially support for a - // new Avx.TestNotZ intrinsic to ensure the codegen remains efficient. - assert(compIsaSupportedDebugOnly(InstructionSet_AVX2)); - - intrinsic = NI_Vector256_op_Inequality; - } - else if (simdSize == 64) - { - assert(IsBaselineVector512IsaSupportedDebugOnly()); - intrinsic = NI_Vector512_op_Inequality; - } - else - { - intrinsic = NI_Vector128_op_Inequality; - } - - op1 = gtNewSimdCmpOpNode(op, simdType, op1, op2, simdBaseJitType, simdSize); - op2 = gtNewZeroConNode(simdType); - - if (simdBaseType == TYP_FLOAT) - { - simdBaseType = TYP_INT; - simdBaseJitType = CORINFO_TYPE_INT; - } - else if (simdBaseType == TYP_DOUBLE) - { - simdBaseType = TYP_LONG; - simdBaseJitType = CORINFO_TYPE_LONG; - } - break; - } - - case GT_NE: - { - if (simdSize == 64) - { - assert(IsBaselineVector512IsaSupportedDebugOnly()); - intrinsic = NI_Vector512_op_Inequality; - } - else if (simdSize == 32) - { - assert(compIsaSupportedDebugOnly(InstructionSet_AVX)); - assert(varTypeIsFloating(simdBaseType) || compIsaSupportedDebugOnly(InstructionSet_AVX2)); - - intrinsic = NI_Vector256_op_Inequality; - } - else - { - intrinsic = NI_Vector128_op_Inequality; - } - break; - } -#elif defined(TARGET_ARM64) - case GT_EQ: - case GT_GE: - case GT_GT: - case GT_LE: - case GT_LT: - { - // We want to generate a comparison along the lines of - // GT_XX(op1, op2).As() != Vector128.Zero - - intrinsic = (simdSize == 8) ? NI_Vector64_op_Inequality : NI_Vector128_op_Inequality; - - op1 = gtNewSimdCmpOpNode(op, simdType, op1, op2, simdBaseJitType, simdSize); - op2 = gtNewZeroConNode(simdType); - - if (simdBaseType == TYP_FLOAT) - { - simdBaseType = TYP_INT; - simdBaseJitType = CORINFO_TYPE_INT; - } - else if (simdBaseType == TYP_DOUBLE) - { - simdBaseType = TYP_LONG; - simdBaseJitType = CORINFO_TYPE_LONG; - } - break; - } - - case GT_NE: - { - intrinsic = (simdSize == 8) ? NI_Vector64_op_Inequality : NI_Vector128_op_Inequality; - break; - } -#else -#error Unsupported platform -#endif // !TARGET_XARCH && !TARGET_ARM64 - - default: - { - unreached(); - } - } - - assert(intrinsic != NI_Illegal); - return gtNewSimdHWIntrinsicNode(type, op1, op2, intrinsic, simdBaseJitType, simdSize); -} - -GenTree* Compiler::gtNewSimdCndSelNode( - var_types type, GenTree* op1, GenTree* op2, GenTree* op3, CorInfoType simdBaseJitType, unsigned simdSize) -{ - assert(IsBaselineSimdIsaSupportedDebugOnly()); - - assert(varTypeIsSIMD(type)); - assert(getSIMDTypeForSize(simdSize) == type); - - assert(op1 != nullptr); - assert(op1->TypeIs(type)); - - assert(op2 != nullptr); - assert(op2->TypeIs(type)); - - assert(op3 != nullptr); - assert(op3->TypeIs(type)); - - var_types simdBaseType = JitType2PreciseVarType(simdBaseJitType); - assert(varTypeIsArithmetic(simdBaseType)); - - NamedIntrinsic intrinsic = NI_Illegal; - -#if defined(TARGET_XARCH) - if (simdSize == 64) - { - assert(canUseEvexEncodingDebugOnly()); - intrinsic = NI_Vector512_ConditionalSelect; - } - else if (simdSize == 32) - { - assert(compIsaSupportedDebugOnly(InstructionSet_AVX)); - intrinsic = NI_Vector256_ConditionalSelect; - } - else - { - intrinsic = NI_Vector128_ConditionalSelect; - } - return gtNewSimdHWIntrinsicNode(type, op1, op2, op3, intrinsic, simdBaseJitType, simdSize); -#elif defined(TARGET_ARM64) - return gtNewSimdHWIntrinsicNode(type, op1, op2, op3, NI_AdvSimd_BitwiseSelect, simdBaseJitType, simdSize); -#else -#error Unsupported platform -#endif // !TARGET_XARCH && !TARGET_ARM64 -} - -//---------------------------------------------------------------------------------------------- -// Compiler::gtNewSimdCreateBroadcastNode: Creates a new simd CreateBroadcast node -// -// Arguments: -// type - The return type of SIMD node being created -// op1 - The value of broadcast to every element of the simd value -// simdBaseJitType - The base JIT type of SIMD type of the intrinsic -// simdSize - The size of the SIMD type of the intrinsic -// -// Returns: -// The created CreateBroadcast node -// -GenTree* Compiler::gtNewSimdCreateBroadcastNode(var_types type, - GenTree* op1, - CorInfoType simdBaseJitType, - unsigned simdSize) -{ - NamedIntrinsic hwIntrinsicID = NI_Vector128_Create; - var_types simdBaseType = JitType2PreciseVarType(simdBaseJitType); - - if (op1->IsIntegralConst() || op1->IsCnsFltOrDbl()) - { - GenTreeVecCon* vecCon = gtNewVconNode(type); - - switch (simdBaseType) - { - case TYP_BYTE: - case TYP_UBYTE: - { - uint8_t cnsVal = static_cast(op1->AsIntConCommon()->IntegralValue()); - - for (unsigned i = 0; i < simdSize; i++) - { - vecCon->gtSimdVal.u8[i] = cnsVal; - } - break; - } - - case TYP_SHORT: - case TYP_USHORT: - { - uint16_t cnsVal = static_cast(op1->AsIntConCommon()->IntegralValue()); - - for (unsigned i = 0; i < (simdSize / 2); i++) - { - vecCon->gtSimdVal.u16[i] = cnsVal; - } - break; - } - - case TYP_INT: - case TYP_UINT: - { - uint32_t cnsVal = static_cast(op1->AsIntConCommon()->IntegralValue()); - - for (unsigned i = 0; i < (simdSize / 4); i++) - { - vecCon->gtSimdVal.u32[i] = cnsVal; - } - break; - } - - case TYP_LONG: - case TYP_ULONG: - { - uint64_t cnsVal = static_cast(op1->AsIntConCommon()->IntegralValue()); - - for (unsigned i = 0; i < (simdSize / 8); i++) - { - vecCon->gtSimdVal.u64[i] = cnsVal; - } - break; - } - - case TYP_FLOAT: - { - float cnsVal = static_cast(op1->AsDblCon()->DconValue()); - - for (unsigned i = 0; i < (simdSize / 4); i++) - { - vecCon->gtSimdVal.f32[i] = cnsVal; - } - break; - } - - case TYP_DOUBLE: - { - double cnsVal = static_cast(op1->AsDblCon()->DconValue()); - - for (unsigned i = 0; i < (simdSize / 8); i++) - { - vecCon->gtSimdVal.f64[i] = cnsVal; - } - break; - } - - default: - { - unreached(); - } - } - - return vecCon; - } - -#if defined(TARGET_XARCH) -#if defined(TARGET_X86) - if (varTypeIsLong(simdBaseType) && !op1->IsIntegralConst()) - { - // TODO-XARCH-CQ: It may be beneficial to emit the movq - // instruction, which takes a 64-bit memory address and - // works on 32-bit x86 systems. - unreached(); - } -#endif // TARGET_X86 - - if (simdSize == 64) - { - hwIntrinsicID = NI_Vector512_Create; - } - else if (simdSize == 32) - { - hwIntrinsicID = NI_Vector256_Create; - } -#elif defined(TARGET_ARM64) - if (simdSize == 8) - { - hwIntrinsicID = NI_Vector64_Create; - } -#else -#error Unsupported platform -#endif // !TARGET_XARCH && !TARGET_ARM64 - - return gtNewSimdHWIntrinsicNode(type, op1, hwIntrinsicID, simdBaseJitType, simdSize); -} - -//---------------------------------------------------------------------------------------------- -// Compiler::gtNewSimdCreateScalarNode: Creates a new simd CreateScalar node -// -// Arguments: -// type - The return type of SIMD node being created -// op1 - The value of element 0 of the simd value -// simdBaseJitType - The base JIT type of SIMD type of the intrinsic -// simdSize - The size of the SIMD type of the intrinsic -// -// Returns: -// The created CreateScalar node -// -GenTree* Compiler::gtNewSimdCreateScalarNode(var_types type, - GenTree* op1, - CorInfoType simdBaseJitType, - unsigned simdSize) -{ - NamedIntrinsic hwIntrinsicID = NI_Vector128_CreateScalar; - var_types simdBaseType = JitType2PreciseVarType(simdBaseJitType); - - if (op1->IsIntegralConst() || op1->IsCnsFltOrDbl()) - { - GenTreeVecCon* vecCon = gtNewVconNode(type); - vecCon->gtSimdVal = {}; - - switch (simdBaseType) - { - case TYP_BYTE: - case TYP_UBYTE: - { - uint8_t cnsVal = static_cast(op1->AsIntConCommon()->IntegralValue()); - vecCon->gtSimdVal.u8[0] = cnsVal; - break; - } - - case TYP_SHORT: - case TYP_USHORT: - { - uint16_t cnsVal = static_cast(op1->AsIntConCommon()->IntegralValue()); - vecCon->gtSimdVal.u16[0] = cnsVal; - break; - } - - case TYP_INT: - case TYP_UINT: - { - uint32_t cnsVal = static_cast(op1->AsIntConCommon()->IntegralValue()); - vecCon->gtSimdVal.u32[0] = cnsVal; - break; - } - - case TYP_LONG: - case TYP_ULONG: - { - uint64_t cnsVal = static_cast(op1->AsIntConCommon()->IntegralValue()); - vecCon->gtSimdVal.u64[0] = cnsVal; - break; - } - - case TYP_FLOAT: - { - float cnsVal = static_cast(op1->AsDblCon()->DconValue()); - vecCon->gtSimdVal.f32[0] = cnsVal; - break; - } - - case TYP_DOUBLE: - { - double cnsVal = static_cast(op1->AsDblCon()->DconValue()); - vecCon->gtSimdVal.f64[0] = cnsVal; - break; - } - - default: - { - unreached(); - } - } - - return vecCon; - } - -#if defined(TARGET_XARCH) -#if defined(TARGET_X86) - if (varTypeIsLong(simdBaseType) && !op1->IsIntegralConst()) - { - // TODO-XARCH-CQ: It may be beneficial to emit the movq - // instruction, which takes a 64-bit memory address and - // works on 32-bit x86 systems. - unreached(); - } -#endif // TARGET_X86 - - if (simdSize == 32) - { - hwIntrinsicID = NI_Vector256_CreateScalar; - } - else if (simdSize == 64) - { - hwIntrinsicID = NI_Vector512_CreateScalar; - } -#elif defined(TARGET_ARM64) - if (simdSize == 8) - { - hwIntrinsicID = (genTypeSize(simdBaseType) == 8) ? NI_Vector64_Create : NI_Vector64_CreateScalar; - } -#else -#error Unsupported platform -#endif // !TARGET_XARCH && !TARGET_ARM64 - - return gtNewSimdHWIntrinsicNode(type, op1, hwIntrinsicID, simdBaseJitType, simdSize); -} - -//---------------------------------------------------------------------------------------------- -// Compiler::gtNewSimdCreateScalarUnsafeNode: Creates a new simd CreateScalarUnsafe node -// -// Arguments: -// type - The return type of SIMD node being created -// op1 - The value of element 0 of the simd value -// simdBaseJitType - The base JIT type of SIMD type of the intrinsic -// simdSize - The size of the SIMD type of the intrinsic -// -// Returns: -// The created CreateScalarUnsafe node -// -// Remarks: -// This API is unsafe as it leaves the upper-bits of the vector undefined -// -GenTree* Compiler::gtNewSimdCreateScalarUnsafeNode(var_types type, - GenTree* op1, - CorInfoType simdBaseJitType, - unsigned simdSize) -{ - NamedIntrinsic hwIntrinsicID = NI_Vector128_CreateScalarUnsafe; - var_types simdBaseType = JitType2PreciseVarType(simdBaseJitType); - - if (op1->IsIntegralConst() || op1->IsCnsFltOrDbl()) - { - GenTreeVecCon* vecCon = gtNewVconNode(type); - - // Since the upper bits are considered non-deterministic and we can therefore - // set them to anything, we broadcast the value. - // - // We do this as it simplifies the logic and allows certain code paths to - // have better codegen, such as for 0, AllBitsSet, or certain small constants - - switch (simdBaseType) - { - case TYP_BYTE: - case TYP_UBYTE: - { - uint8_t cnsVal = static_cast(op1->AsIntConCommon()->IntegralValue()); - - for (unsigned i = 0; i < simdSize; i++) - { - vecCon->gtSimdVal.u8[i] = cnsVal; - } - break; - } - - case TYP_SHORT: - case TYP_USHORT: - { - uint16_t cnsVal = static_cast(op1->AsIntConCommon()->IntegralValue()); - - for (unsigned i = 0; i < (simdSize / 2); i++) - { - vecCon->gtSimdVal.u16[i] = cnsVal; - } - break; - } - - case TYP_INT: - case TYP_UINT: - { - uint32_t cnsVal = static_cast(op1->AsIntConCommon()->IntegralValue()); - - for (unsigned i = 0; i < (simdSize / 4); i++) - { - vecCon->gtSimdVal.u32[i] = cnsVal; - } - break; - } - - case TYP_LONG: - case TYP_ULONG: - { - uint64_t cnsVal = static_cast(op1->AsIntConCommon()->IntegralValue()); - - for (unsigned i = 0; i < (simdSize / 8); i++) - { - vecCon->gtSimdVal.u64[i] = cnsVal; - } - break; - } - - case TYP_FLOAT: - { - float cnsVal = static_cast(op1->AsDblCon()->DconValue()); - - for (unsigned i = 0; i < (simdSize / 4); i++) - { - vecCon->gtSimdVal.f32[i] = cnsVal; - } - break; - } - - case TYP_DOUBLE: - { - double cnsVal = static_cast(op1->AsDblCon()->DconValue()); - - for (unsigned i = 0; i < (simdSize / 8); i++) - { - vecCon->gtSimdVal.f64[i] = cnsVal; - } - break; - } - - default: - { - unreached(); - } - } - - return vecCon; - } - -#if defined(TARGET_XARCH) -#if defined(TARGET_X86) - if (varTypeIsLong(simdBaseType) && !op1->IsIntegralConst()) - { - // TODO-XARCH-CQ: It may be beneficial to emit the movq - // instruction, which takes a 64-bit memory address and - // works on 32-bit x86 systems. - unreached(); - } -#endif // TARGET_X86 - - if (simdSize == 32) - { - hwIntrinsicID = NI_Vector256_CreateScalarUnsafe; - } - else if (simdSize == 64) - { - hwIntrinsicID = NI_Vector512_CreateScalarUnsafe; - } -#elif defined(TARGET_ARM64) - if (simdSize == 8) - { - hwIntrinsicID = (genTypeSize(simdBaseType) == 8) ? NI_Vector64_Create : NI_Vector64_CreateScalarUnsafe; - } -#else -#error Unsupported platform -#endif // !TARGET_XARCH && !TARGET_ARM64 - - return gtNewSimdHWIntrinsicNode(type, op1, hwIntrinsicID, simdBaseJitType, simdSize); -} - -//---------------------------------------------------------------------------------------------- -// Compiler::gtNewSimdCreateSequenceNode: Creates a new simd CreateSequence node -// -// Arguments: -// type - The return type of SIMD node being created -// op1 - The starting value -// op2 - The step value -// simdBaseJitType - The base JIT type of SIMD type of the intrinsic -// simdSize - The size of the SIMD type of the intrinsic -// -// Returns: -// The created CreateSequence node -// -GenTree* Compiler::gtNewSimdCreateSequenceNode( - var_types type, GenTree* op1, GenTree* op2, CorInfoType simdBaseJitType, unsigned simdSize) -{ - // This effectively doees: (Indices * op2) + Create(op1) - // - // When both op2 and op1 are constant we can fully fold this to a constant. Additionally, - // if only op2 is a constant we can simplify the computation by a lot. However, if only op1 - // is constant than there isn't any real optimization we can do and we need the full computation. - - assert(varTypeIsSIMD(type)); - assert(getSIMDTypeForSize(simdSize) == type); - - var_types simdBaseType = JitType2PreciseVarType(simdBaseJitType); - assert(varTypeIsArithmetic(simdBaseType)); - - GenTree* result = nullptr; - bool isPartial = true; - - if (op2->OperIsConst()) - { - GenTreeVecCon* vcon = gtNewVconNode(type); - uint32_t simdLength = getSIMDVectorLength(simdSize, simdBaseType); - - switch (simdBaseType) - { - case TYP_BYTE: - case TYP_UBYTE: - { - uint8_t start = 0; - - if (op1->OperIsConst()) - { - assert(op1->IsIntegralConst()); - start = static_cast(op1->AsIntConCommon()->IntegralValue()); - isPartial = false; - } - - assert(op2->IsIntegralConst()); - uint8_t step = static_cast(op2->AsIntConCommon()->IntegralValue()); - - for (uint32_t index = 0; index < simdLength; index++) - { - vcon->gtSimdVal.u8[index] = static_cast((index * step) + start); - } - break; - } - - case TYP_SHORT: - case TYP_USHORT: - { - uint16_t start = 0; - - if (op1->OperIsConst()) - { - assert(op1->IsIntegralConst()); - start = static_cast(op1->AsIntConCommon()->IntegralValue()); - isPartial = false; - } - - assert(op2->IsIntegralConst()); - uint16_t step = static_cast(op2->AsIntConCommon()->IntegralValue()); - - for (uint32_t index = 0; index < simdLength; index++) - { - vcon->gtSimdVal.u16[index] = static_cast((index * step) + start); - } - break; - } - - case TYP_INT: - case TYP_UINT: - { - uint32_t start = 0; - - if (op1->OperIsConst()) - { - assert(op1->IsIntegralConst()); - start = static_cast(op1->AsIntConCommon()->IntegralValue()); - isPartial = false; - } - - assert(op2->IsIntegralConst()); - uint32_t step = static_cast(op2->AsIntConCommon()->IntegralValue()); - - for (uint32_t index = 0; index < simdLength; index++) - { - vcon->gtSimdVal.u32[index] = static_cast((index * step) + start); - } - break; - } - - case TYP_LONG: - case TYP_ULONG: - { - uint64_t start = 0; - - if (op1->OperIsConst()) - { - assert(op1->IsIntegralConst()); - start = static_cast(op1->AsIntConCommon()->IntegralValue()); - isPartial = false; - } - - assert(op2->IsIntegralConst()); - uint64_t step = static_cast(op2->AsIntConCommon()->IntegralValue()); - - for (uint32_t index = 0; index < simdLength; index++) - { - vcon->gtSimdVal.u64[index] = static_cast((index * step) + start); - } - break; - } - - case TYP_FLOAT: - { - float start = 0; - - if (op1->OperIsConst()) - { - assert(op1->IsCnsFltOrDbl()); - start = static_cast(op1->AsDblCon()->DconValue()); - isPartial = false; - } - - assert(op2->IsCnsFltOrDbl()); - float step = static_cast(op2->AsDblCon()->DconValue()); - - for (uint32_t index = 0; index < simdLength; index++) - { - vcon->gtSimdVal.f32[index] = static_cast((index * step) + start); - } - break; - } - - case TYP_DOUBLE: - { - double start = 0; - - if (op1->OperIsConst()) - { - assert(op1->IsCnsFltOrDbl()); - start = static_cast(op1->AsDblCon()->DconValue()); - isPartial = false; - } - - assert(op2->IsCnsFltOrDbl()); - double step = static_cast(op2->AsDblCon()->DconValue()); - - for (uint32_t index = 0; index < simdLength; index++) - { - vcon->gtSimdVal.f64[index] = static_cast((index * step) + start); - } - break; - } - - default: - { - unreached(); - } - } - - result = vcon; - } - else - { - GenTree* indices = gtNewSimdGetIndicesNode(type, simdBaseJitType, simdSize); - result = gtNewSimdBinOpNode(GT_MUL, type, indices, op2, simdBaseJitType, simdSize); - } - - if (isPartial) - { - GenTree* start = gtNewSimdCreateBroadcastNode(type, op1, simdBaseJitType, simdSize); - result = gtNewSimdBinOpNode(GT_ADD, type, result, start, simdBaseJitType, simdSize); - } - - return result; -} - -GenTree* Compiler::gtNewSimdDotProdNode( - var_types type, GenTree* op1, GenTree* op2, CorInfoType simdBaseJitType, unsigned simdSize) -{ - assert(IsBaselineSimdIsaSupportedDebugOnly()); - - var_types simdType = getSIMDTypeForSize(simdSize); - assert(varTypeIsSIMD(simdType)); - - assert(op1 != nullptr); - assert(op1->TypeIs(simdType)); - - assert(op2 != nullptr); - assert(op2->TypeIs(simdType)); - - var_types simdBaseType = JitType2PreciseVarType(simdBaseJitType); - assert(varTypeIsSIMD(type)); - - NamedIntrinsic intrinsic = NI_Illegal; - -#if defined(TARGET_XARCH) - assert(!varTypeIsByte(simdBaseType) && !varTypeIsLong(simdBaseType)); - assert(simdSize != 64); - - if (simdSize == 32) - { - assert(varTypeIsFloating(simdBaseType) || compIsaSupportedDebugOnly(InstructionSet_AVX2)); - intrinsic = NI_Vector256_Dot; - } - else - { - assert(((simdBaseType != TYP_INT) && (simdBaseType != TYP_UINT)) || - compIsaSupportedDebugOnly(InstructionSet_SSE41)); - intrinsic = NI_Vector128_Dot; - } -#elif defined(TARGET_ARM64) - assert(!varTypeIsLong(simdBaseType)); - intrinsic = (simdSize == 8) ? NI_Vector64_Dot : NI_Vector128_Dot; -#else -#error Unsupported platform -#endif // !TARGET_XARCH && !TARGET_ARM64 - - assert(intrinsic != NI_Illegal); - return gtNewSimdHWIntrinsicNode(type, op1, op2, intrinsic, simdBaseJitType, simdSize); -} - -GenTree* Compiler::gtNewSimdFloorNode(var_types type, GenTree* op1, CorInfoType simdBaseJitType, unsigned simdSize) -{ - assert(IsBaselineSimdIsaSupportedDebugOnly()); - - assert(varTypeIsSIMD(type)); - assert(getSIMDTypeForSize(simdSize) == type); - - assert(op1 != nullptr); - assert(op1->TypeIs(type)); - - var_types simdBaseType = JitType2PreciseVarType(simdBaseJitType); - assert(varTypeIsFloating(simdBaseType)); - - NamedIntrinsic intrinsic = NI_Illegal; - -#if defined(TARGET_XARCH) - if (simdSize == 32) - { - intrinsic = NI_AVX_Floor; - } - else if (simdSize == 64) - { - assert(compIsaSupportedDebugOnly(InstructionSet_AVX512F)); - GenTree* op2 = gtNewIconNode(static_cast(FloatRoundingMode::ToNegativeInfinity)); - return gtNewSimdHWIntrinsicNode(type, op1, op2, NI_AVX512F_RoundScale, simdBaseJitType, simdSize); - } - else - { - assert(compIsaSupportedDebugOnly(InstructionSet_SSE41)); - intrinsic = NI_SSE41_Floor; - } -#elif defined(TARGET_ARM64) - if (simdBaseType == TYP_DOUBLE) - { - intrinsic = (simdSize == 8) ? NI_AdvSimd_FloorScalar : NI_AdvSimd_Arm64_Floor; - } - else - { - intrinsic = NI_AdvSimd_Floor; - } -#else -#error Unsupported platform -#endif // !TARGET_XARCH && !TARGET_ARM64 - - assert(intrinsic != NI_Illegal); - return gtNewSimdHWIntrinsicNode(type, op1, intrinsic, simdBaseJitType, simdSize); -} - -GenTree* Compiler::gtNewSimdFmaNode( - var_types type, GenTree* op1, GenTree* op2, GenTree* op3, CorInfoType simdBaseJitType, unsigned simdSize) -{ - assert(varTypeIsSIMD(type)); - assert(getSIMDTypeForSize(simdSize) == type); - - assert(op1 != nullptr); - assert(op1->TypeIs(type)); - - assert(op2 != nullptr); - assert(op2->TypeIs(type)); - - assert(op3 != nullptr); - assert(op3->TypeIs(type)); - - var_types simdBaseType = JitType2PreciseVarType(simdBaseJitType); - assert(varTypeIsFloating(simdBaseType)); - - NamedIntrinsic intrinsic = NI_Illegal; - -#if defined(TARGET_XARCH) - if (simdSize == 64) - { - assert(compIsaSupportedDebugOnly(InstructionSet_AVX512F)); - intrinsic = NI_AVX512F_FusedMultiplyAdd; - } - else - { - assert(compIsaSupportedDebugOnly(InstructionSet_FMA)); - intrinsic = NI_FMA_MultiplyAdd; - } -#elif defined(TARGET_ARM64) - assert(IsBaselineSimdIsaSupportedDebugOnly()); - - if (simdBaseType == TYP_DOUBLE) - { - intrinsic = (simdSize == 8) ? NI_AdvSimd_FusedMultiplyAddScalar : NI_AdvSimd_Arm64_FusedMultiplyAdd; - } - else - { - intrinsic = NI_AdvSimd_FusedMultiplyAdd; - } - - // AdvSimd.FusedMultiplyAdd expects (addend, left, right), while the APIs take (left, right, addend) - // We expect op1 and op2 to have already been spilled - - std::swap(op1, op3); -#else -#error Unsupported platform -#endif // !TARGET_XARCH && !TARGET_ARM64 - - assert(intrinsic != NI_Illegal); - return gtNewSimdHWIntrinsicNode(type, op1, op2, op3, intrinsic, simdBaseJitType, simdSize); -} - -GenTree* Compiler::gtNewSimdGetElementNode( - var_types type, GenTree* op1, GenTree* op2, CorInfoType simdBaseJitType, unsigned simdSize) -{ - NamedIntrinsic intrinsicId = NI_Vector128_GetElement; - var_types simdBaseType = JitType2PreciseVarType(simdBaseJitType); - - assert(varTypeIsArithmetic(simdBaseType)); - -#if defined(TARGET_XARCH) - bool useToScalar = op2->IsIntegralConst(0); - -#if defined(TARGET_X86) - // We handle decomposition via GetElement for simplicity - useToScalar &= !varTypeIsLong(simdBaseType); -#endif // TARGET_X86 - - if (useToScalar) - { - return gtNewSimdToScalarNode(type, op1, simdBaseJitType, simdSize); - } - - switch (simdBaseType) - { - case TYP_BYTE: - case TYP_UBYTE: - case TYP_INT: - case TYP_UINT: - case TYP_LONG: - case TYP_ULONG: - { - // Using software fallback if simdBaseType is not supported by hardware - assert(compIsaSupportedDebugOnly(InstructionSet_SSE41)); - break; - } - - case TYP_DOUBLE: - case TYP_FLOAT: - case TYP_SHORT: - case TYP_USHORT: - { - assert(compIsaSupportedDebugOnly(InstructionSet_SSE2)); - break; - } - - default: - { - unreached(); - } - } - - if (simdSize == 64) - { - intrinsicId = NI_Vector512_GetElement; - } - else if (simdSize == 32) - { - intrinsicId = NI_Vector256_GetElement; - } -#elif defined(TARGET_ARM64) - if (op2->IsIntegralConst(0)) - { - return gtNewSimdToScalarNode(type, op1, simdBaseJitType, simdSize); - } - - if (simdSize == 8) - { - intrinsicId = NI_Vector64_GetElement; - } -#else -#error Unsupported platform -#endif // !TARGET_XARCH && !TARGET_ARM64 - - int immUpperBound = getSIMDVectorLength(simdSize, simdBaseType) - 1; - bool rangeCheckNeeded = !op2->OperIsConst(); - - if (!rangeCheckNeeded) - { - ssize_t imm8 = op2->AsIntCon()->IconValue(); - rangeCheckNeeded = (imm8 < 0) || (imm8 > immUpperBound); - } - - if (rangeCheckNeeded) - { - op2 = addRangeCheckForHWIntrinsic(op2, 0, immUpperBound); - } - - return gtNewSimdHWIntrinsicNode(type, op1, op2, intrinsicId, simdBaseJitType, simdSize); -} - -//---------------------------------------------------------------------------------------------- -// Compiler::gtNewSimdGetIndicesNode: Creates a new simd get_Indices node -// -// Arguments: -// type - The return type of SIMD node being created -// simdBaseJitType - The base JIT type of SIMD type of the intrinsic -// simdSize - The size of the SIMD type of the intrinsic -// -// Returns: -// The created get_Indices node -// -GenTree* Compiler::gtNewSimdGetIndicesNode(var_types type, CorInfoType simdBaseJitType, unsigned simdSize) -{ - assert(varTypeIsSIMD(type)); - assert(getSIMDTypeForSize(simdSize) == type); - - var_types simdBaseType = JitType2PreciseVarType(simdBaseJitType); - assert(varTypeIsArithmetic(simdBaseType)); - - GenTreeVecCon* indices = gtNewVconNode(type); - uint32_t simdLength = getSIMDVectorLength(simdSize, simdBaseType); - - switch (simdBaseType) - { - case TYP_BYTE: - case TYP_UBYTE: - { - for (uint32_t index = 0; index < simdLength; index++) - { - indices->gtSimdVal.u8[index] = static_cast(index); - } - break; - } - - case TYP_SHORT: - case TYP_USHORT: - { - for (uint32_t index = 0; index < simdLength; index++) - { - indices->gtSimdVal.u16[index] = static_cast(index); - } - break; - } - - case TYP_INT: - case TYP_UINT: - { - for (uint32_t index = 0; index < simdLength; index++) - { - indices->gtSimdVal.u32[index] = static_cast(index); - } - break; - } - - case TYP_LONG: - case TYP_ULONG: - { - for (uint32_t index = 0; index < simdLength; index++) - { - indices->gtSimdVal.u64[index] = static_cast(index); - } - break; - } - - case TYP_FLOAT: - { - for (uint32_t index = 0; index < simdLength; index++) - { - indices->gtSimdVal.f32[index] = static_cast(index); - } - break; - } - - case TYP_DOUBLE: - { - for (uint32_t index = 0; index < simdLength; index++) - { - indices->gtSimdVal.f64[index] = static_cast(index); - } - break; - } - - default: - { - unreached(); - } - } - - return indices; -} - -GenTree* Compiler::gtNewSimdGetLowerNode(var_types type, GenTree* op1, CorInfoType simdBaseJitType, unsigned simdSize) -{ - var_types simdBaseType = JitType2PreciseVarType(simdBaseJitType); - assert(varTypeIsArithmetic(simdBaseType)); - - NamedIntrinsic intrinsicId = NI_Illegal; - -#if defined(TARGET_XARCH) - if (simdSize == 32) - { - assert(type == TYP_SIMD16); - intrinsicId = NI_Vector256_GetLower; - } - else - { - assert((type == TYP_SIMD32) && (simdSize == 64)); - intrinsicId = NI_Vector512_GetLower; - } -#elif defined(TARGET_ARM64) - assert((type == TYP_SIMD8) && (simdSize == 16)); - intrinsicId = NI_Vector128_GetLower; -#else -#error Unsupported platform -#endif // !TARGET_XARCH && !TARGET_ARM64 - - assert(intrinsicId != NI_Illegal); - return gtNewSimdHWIntrinsicNode(type, op1, intrinsicId, simdBaseJitType, simdSize); -} - -GenTree* Compiler::gtNewSimdGetUpperNode(var_types type, GenTree* op1, CorInfoType simdBaseJitType, unsigned simdSize) -{ - var_types simdBaseType = JitType2PreciseVarType(simdBaseJitType); - assert(varTypeIsArithmetic(simdBaseType)); - - NamedIntrinsic intrinsicId = NI_Illegal; - -#if defined(TARGET_XARCH) - if (simdSize == 32) - { - assert(type == TYP_SIMD16); - intrinsicId = NI_Vector256_GetUpper; - } - else - { - assert((type == TYP_SIMD32) && (simdSize == 64)); - intrinsicId = NI_Vector512_GetUpper; - } -#elif defined(TARGET_ARM64) - assert((type == TYP_SIMD8) && (simdSize == 16)); - intrinsicId = NI_Vector128_GetUpper; -#else -#error Unsupported platform -#endif // !TARGET_XARCH && !TARGET_ARM64 - - assert(intrinsicId != NI_Illegal); - return gtNewSimdHWIntrinsicNode(type, op1, intrinsicId, simdBaseJitType, simdSize); -} - -//---------------------------------------------------------------------------------------------- -// Compiler::gtNewSimdIsNaNNode: Creates a new simd IsNaN node -// -// Arguments: -// type - The return type of SIMD node being created -// op1 - The vector to check for NaNs -// simdBaseJitType - The base JIT type of SIMD type of the intrinsic -// simdSize - The size of the SIMD type of the intrinsic -// -// Returns: -// The created IsNaN node -// -GenTree* Compiler::gtNewSimdIsNaNNode(var_types type, GenTree* op1, CorInfoType simdBaseJitType, unsigned simdSize) -{ - assert(IsBaselineSimdIsaSupportedDebugOnly()); - - assert(varTypeIsSIMD(type)); - assert(getSIMDTypeForSize(simdSize) == type); - - assert(op1 != nullptr); - assert(op1->TypeIs(type)); - - var_types simdBaseType = JitType2PreciseVarType(simdBaseJitType); - assert(varTypeIsArithmetic(simdBaseType)); - - if (varTypeIsFloating(simdBaseType)) - { - GenTree* op1Dup = fgMakeMultiUse(&op1); - return gtNewSimdCmpOpNode(GT_NE, type, op1, op1Dup, simdBaseJitType, simdSize); - } - return gtNewZeroConNode(type); -} - -//---------------------------------------------------------------------------------------------- -// Compiler::gtNewSimdIsNegativeNode: Creates a new simd IsNegative node -// -// Arguments: -// type - The return type of SIMD node being created -// op1 - The vector to check for Negatives -// simdBaseJitType - The base JIT type of SIMD type of the intrinsic -// simdSize - The size of the SIMD type of the intrinsic -// -// Returns: -// The created IsNegative node -// -GenTree* Compiler::gtNewSimdIsNegativeNode(var_types type, GenTree* op1, CorInfoType simdBaseJitType, unsigned simdSize) -{ - assert(IsBaselineSimdIsaSupportedDebugOnly()); - - assert(varTypeIsSIMD(type)); - assert(getSIMDTypeForSize(simdSize) == type); - - assert(op1 != nullptr); - assert(op1->TypeIs(type)); - - if (simdBaseJitType == CORINFO_TYPE_FLOAT) - { - simdBaseJitType = CORINFO_TYPE_INT; - } - else if (simdBaseJitType == CORINFO_TYPE_DOUBLE) - { - simdBaseJitType = CORINFO_TYPE_LONG; - } - - var_types simdBaseType = JitType2PreciseVarType(simdBaseJitType); - assert(varTypeIsIntegral(simdBaseType)); - - if (varTypeIsUnsigned(simdBaseType)) - { - return gtNewZeroConNode(type); - } - return gtNewSimdCmpOpNode(GT_LT, type, op1, gtNewZeroConNode(type), simdBaseJitType, simdSize); -} - -//---------------------------------------------------------------------------------------------- -// Compiler::gtNewSimdIsPositiveNode: Creates a new simd IsPositive node -// -// Arguments: -// type - The return type of SIMD node being created -// op1 - The vector to check for Positives -// simdBaseJitType - The base JIT type of SIMD type of the intrinsic -// simdSize - The size of the SIMD type of the intrinsic -// -// Returns: -// The created IsPositive node -// -GenTree* Compiler::gtNewSimdIsPositiveNode(var_types type, GenTree* op1, CorInfoType simdBaseJitType, unsigned simdSize) -{ - assert(IsBaselineSimdIsaSupportedDebugOnly()); - - assert(varTypeIsSIMD(type)); - assert(getSIMDTypeForSize(simdSize) == type); - - assert(op1 != nullptr); - assert(op1->TypeIs(type)); - - if (simdBaseJitType == CORINFO_TYPE_FLOAT) - { - simdBaseJitType = CORINFO_TYPE_INT; - } - else if (simdBaseJitType == CORINFO_TYPE_DOUBLE) - { - simdBaseJitType = CORINFO_TYPE_LONG; - } - - var_types simdBaseType = JitType2PreciseVarType(simdBaseJitType); - assert(varTypeIsIntegral(simdBaseType)); - - if (varTypeIsUnsigned(simdBaseType)) - { - return gtNewAllBitsSetConNode(type); - } - return gtNewSimdCmpOpNode(GT_GE, type, op1, gtNewZeroConNode(type), simdBaseJitType, simdSize); -} - -//---------------------------------------------------------------------------------------------- -// Compiler::gtNewSimdIsPositiveInfinityNode: Creates a new simd IsPositiveInfinity node -// -// Arguments: -// type - The return type of SIMD node being created -// op1 - The vector to check for PositiveInfinities -// simdBaseJitType - The base JIT type of SIMD type of the intrinsic -// simdSize - The size of the SIMD type of the intrinsic -// -// Returns: -// The created IsPositiveInfinity node -// -GenTree* Compiler::gtNewSimdIsPositiveInfinityNode(var_types type, - GenTree* op1, - CorInfoType simdBaseJitType, - unsigned simdSize) -{ - assert(IsBaselineSimdIsaSupportedDebugOnly()); - - assert(varTypeIsSIMD(type)); - assert(getSIMDTypeForSize(simdSize) == type); - - assert(op1 != nullptr); - assert(op1->TypeIs(type)); - - var_types simdBaseType = JitType2PreciseVarType(simdBaseJitType); - assert(varTypeIsArithmetic(simdBaseType)); - - if (varTypeIsFloating(simdBaseType)) - { - double infinity = BitOperations::UInt64BitsToDouble(0x7FF0000000000000); - GenTree* cnsNode = gtNewDconNode(infinity, simdBaseType); - cnsNode = gtNewSimdCreateBroadcastNode(type, cnsNode, simdBaseJitType, simdSize); - return gtNewSimdCmpOpNode(GT_EQ, type, op1, cnsNode, simdBaseJitType, simdSize); - } - return gtNewZeroConNode(type); -} - -//---------------------------------------------------------------------------------------------- -// Compiler::gtNewSimdIsZeroNode: Creates a new simd IsZero node -// -// Arguments: -// type - The return type of SIMD node being created -// op1 - The vector to check for Zeroes -// simdBaseJitType - The base JIT type of SIMD type of the intrinsic -// simdSize - The size of the SIMD type of the intrinsic -// -// Returns: -// The created IsZero node -// -GenTree* Compiler::gtNewSimdIsZeroNode(var_types type, GenTree* op1, CorInfoType simdBaseJitType, unsigned simdSize) -{ - assert(IsBaselineSimdIsaSupportedDebugOnly()); - - assert(varTypeIsSIMD(type)); - assert(getSIMDTypeForSize(simdSize) == type); - - assert(op1 != nullptr); - assert(op1->TypeIs(type)); - - var_types simdBaseType = JitType2PreciseVarType(simdBaseJitType); - assert(varTypeIsArithmetic(simdBaseType)); - - return gtNewSimdCmpOpNode(GT_EQ, type, op1, gtNewZeroConNode(type), simdBaseJitType, simdSize); -} - -//---------------------------------------------------------------------------------------------- -// Compiler::gtNewSimdLoadNode: Creates a new simd Load node -// -// Arguments: -// type - The return type of SIMD node being created -// op1 - The address of the value to be loaded -// simdBaseJitType - The base JIT type of SIMD type of the intrinsic -// simdSize - The size of the SIMD type of the intrinsic -// -// Returns: -// The created Load node -// -GenTree* Compiler::gtNewSimdLoadNode(var_types type, GenTree* op1, CorInfoType simdBaseJitType, unsigned simdSize) -{ - assert(varTypeIsSIMD(type)); - assert(getSIMDTypeForSize(simdSize) == type); - - assert(op1 != nullptr); - - var_types simdBaseType = JitType2PreciseVarType(simdBaseJitType); - assert(varTypeIsArithmetic(simdBaseType)); - - return gtNewIndir(type, op1); -} - -//---------------------------------------------------------------------------------------------- -// Compiler::gtNewSimdLoadAlignedNode: Creates a new simd LoadAligned node -// -// Arguments: -// type - The return type of SIMD node being created -// op1 - The address of the value to be loaded -// simdBaseJitType - The base JIT type of SIMD type of the intrinsic -// simdSize - The size of the SIMD type of the intrinsic -// -// Returns: -// The created LoadAligned node -// -GenTree* Compiler::gtNewSimdLoadAlignedNode(var_types type, - GenTree* op1, - CorInfoType simdBaseJitType, - unsigned simdSize) -{ -#if defined(TARGET_XARCH) - assert(IsBaselineSimdIsaSupportedDebugOnly()); - - assert(varTypeIsSIMD(type)); - assert(getSIMDTypeForSize(simdSize) == type); - - assert(op1 != nullptr); - - var_types simdBaseType = JitType2PreciseVarType(simdBaseJitType); - assert(varTypeIsArithmetic(simdBaseType)); - - NamedIntrinsic intrinsic = NI_Illegal; - - if (simdSize == 64) - { - assert(compIsaSupportedDebugOnly(InstructionSet_AVX512F)); - intrinsic = NI_AVX512F_LoadAlignedVector512; - } - else if (simdSize == 32) - { - assert(compIsaSupportedDebugOnly(InstructionSet_AVX)); - intrinsic = NI_AVX_LoadAlignedVector256; - } - else if (simdBaseType != TYP_FLOAT) - { - intrinsic = NI_SSE2_LoadAlignedVector128; - } - else - { - intrinsic = NI_SSE_LoadAlignedVector128; - } - - assert(intrinsic != NI_Illegal); - return gtNewSimdHWIntrinsicNode(type, op1, intrinsic, simdBaseJitType, simdSize); -#elif defined(TARGET_ARM64) - // ARM64 doesn't have aligned loads, but aligned loads are only validated to be - // aligned when optimizations are disable, so only skip the intrinsic handling - // if optimizations are enabled - - assert(opts.OptimizationEnabled()); - return gtNewSimdLoadNode(type, op1, simdBaseJitType, simdSize); -#else -#error Unsupported platform -#endif // !TARGET_XARCH && !TARGET_ARM64 -} - -//---------------------------------------------------------------------------------------------- -// Compiler::gtNewSimdLoadNonTemporalNode: Creates a new simd LoadNonTemporal node -// -// Arguments: -// type - The return type of SIMD node being created -// op1 - The address of the value to be loaded -// simdBaseJitType - The base JIT type of SIMD type of the intrinsic -// simdSize - The size of the SIMD type of the intrinsic -// -// Returns: -// The created LoadNonTemporal node -// -GenTree* Compiler::gtNewSimdLoadNonTemporalNode(var_types type, - GenTree* op1, - CorInfoType simdBaseJitType, - unsigned simdSize) -{ -#if defined(TARGET_XARCH) - assert(IsBaselineSimdIsaSupportedDebugOnly()); - - assert(varTypeIsSIMD(type)); - assert(getSIMDTypeForSize(simdSize) == type); - - assert(op1 != nullptr); - - var_types simdBaseType = JitType2PreciseVarType(simdBaseJitType); - assert(varTypeIsArithmetic(simdBaseType)); - - NamedIntrinsic intrinsic = NI_Illegal; - bool isNonTemporal = false; - - // We don't guarantee a non-temporal load will actually occur, so fallback - // to regular aligned loads if the required ISA isn't supported. - - if (simdSize == 32) - { - if (compOpportunisticallyDependsOn(InstructionSet_AVX2)) - { - intrinsic = NI_AVX2_LoadAlignedVector256NonTemporal; - isNonTemporal = true; - } - else - { - assert(compIsaSupportedDebugOnly(InstructionSet_AVX)); - intrinsic = NI_AVX_LoadAlignedVector256; - } - } - else if (simdSize == 64) - { - if (compOpportunisticallyDependsOn(InstructionSet_AVX512F)) - { - intrinsic = NI_AVX512F_LoadAlignedVector512NonTemporal; - isNonTemporal = true; - } - } - else if (compOpportunisticallyDependsOn(InstructionSet_SSE41)) - { - intrinsic = NI_SSE41_LoadAlignedVector128NonTemporal; - isNonTemporal = true; - } - else if (simdBaseType != TYP_FLOAT) - { - intrinsic = NI_SSE2_LoadAlignedVector128; - } - else - { - intrinsic = NI_SSE_LoadAlignedVector128; - } - - if (isNonTemporal) - { - // float and double don't have actual instructions for non-temporal loads - // so we'll just use the equivalent integer instruction instead. - - if (simdBaseType == TYP_FLOAT) - { - simdBaseJitType = CORINFO_TYPE_INT; - } - else if (simdBaseType == TYP_DOUBLE) - { - simdBaseJitType = CORINFO_TYPE_LONG; - } - } - - assert(intrinsic != NI_Illegal); - return gtNewSimdHWIntrinsicNode(type, op1, intrinsic, simdBaseJitType, simdSize); -#elif defined(TARGET_ARM64) - // ARM64 doesn't have aligned loads, but aligned loads are only validated to be - // aligned when optimizations are disable, so only skip the intrinsic handling - // if optimizations are enabled - - assert(opts.OptimizationEnabled()); - return gtNewSimdLoadNode(type, op1, simdBaseJitType, simdSize); -#else -#error Unsupported platform -#endif // !TARGET_XARCH && !TARGET_ARM64 -} - -GenTree* Compiler::gtNewSimdMaxNode( - var_types type, GenTree* op1, GenTree* op2, CorInfoType simdBaseJitType, unsigned simdSize) -{ - assert(IsBaselineSimdIsaSupportedDebugOnly()); - - assert(varTypeIsSIMD(type)); - assert(getSIMDTypeForSize(simdSize) == type); - - assert(op1 != nullptr); - assert(op1->TypeIs(type)); - - assert(op2 != nullptr); - assert(op2->TypeIs(type)); - - var_types simdBaseType = JitType2PreciseVarType(simdBaseJitType); - assert(varTypeIsArithmetic(simdBaseType)); - -#if defined(TARGET_XARCH) - if (varTypeIsFloating(simdBaseType)) - { - GenTree* op1Dup1 = fgMakeMultiUse(&op1); - GenTree* op1Dup2 = gtCloneExpr(op1Dup1); - GenTree* op1Dup3 = gtCloneExpr(op1Dup2); - - GenTree* op2Dup1 = fgMakeMultiUse(&op2); - GenTree* op2Dup2 = gtCloneExpr(op2Dup1); - GenTree* op2Dup3 = gtCloneExpr(op2Dup2); - - GenTree* equalsMask = gtNewSimdCmpOpNode(GT_EQ, type, op1, op2, simdBaseJitType, simdSize); - GenTree* isNegativeMask = gtNewSimdIsNegativeNode(type, op2Dup1, simdBaseJitType, simdSize); - GenTree* isNaNMask = gtNewSimdIsNaNNode(type, op1Dup1, simdBaseJitType, simdSize); - GenTree* lessThanMask = gtNewSimdCmpOpNode(GT_LT, type, op2Dup2, op1Dup2, simdBaseJitType, simdSize); - - GenTree* mask = gtNewSimdBinOpNode(GT_AND, type, equalsMask, isNegativeMask, simdBaseJitType, simdSize); - mask = gtNewSimdBinOpNode(GT_OR, type, mask, isNaNMask, simdBaseJitType, simdSize); - mask = gtNewSimdBinOpNode(GT_OR, type, mask, lessThanMask, simdBaseJitType, simdSize); - - return gtNewSimdCndSelNode(type, mask, op1Dup3, op2Dup3, simdBaseJitType, simdSize); - } -#endif // TARGET_XARCH - - return gtNewSimdMaxNativeNode(type, op1, op2, simdBaseJitType, simdSize); -} - -GenTree* Compiler::gtNewSimdMaxNativeNode( - var_types type, GenTree* op1, GenTree* op2, CorInfoType simdBaseJitType, unsigned simdSize) -{ - assert(IsBaselineSimdIsaSupportedDebugOnly()); - - assert(varTypeIsSIMD(type)); - assert(getSIMDTypeForSize(simdSize) == type); - - assert(op1 != nullptr); - assert(op1->TypeIs(type)); - - assert(op2 != nullptr); - assert(op2->TypeIs(type)); - - var_types simdBaseType = JitType2PreciseVarType(simdBaseJitType); - assert(varTypeIsArithmetic(simdBaseType)); - - NamedIntrinsic intrinsic = NI_Illegal; - -#if defined(TARGET_XARCH) - if (simdSize == 32) - { - assert(compIsaSupportedDebugOnly(InstructionSet_AVX)); - - if (varTypeIsFloating(simdBaseType)) - { - intrinsic = NI_AVX_Max; - } - else - { - assert(compIsaSupportedDebugOnly(InstructionSet_AVX2)); - - if (!varTypeIsLong(simdBaseType)) - { - intrinsic = NI_AVX2_Max; - } - else if (compOpportunisticallyDependsOn(InstructionSet_AVX10v1)) - { - intrinsic = NI_AVX10v1_Max; - } - else if (compOpportunisticallyDependsOn(InstructionSet_AVX512F_VL)) - { - intrinsic = NI_AVX512F_VL_Max; - } - } - } - else if (simdSize == 64) - { - if (varTypeIsSmall(simdBaseType)) - { - assert(compIsaSupportedDebugOnly(InstructionSet_AVX512BW)); - intrinsic = NI_AVX512BW_Max; - } - else - { - assert(compIsaSupportedDebugOnly(InstructionSet_AVX512F)); - intrinsic = NI_AVX512F_Max; - } - } - else - { - switch (simdBaseType) - { - case TYP_BYTE: - case TYP_USHORT: - { - if (compOpportunisticallyDependsOn(InstructionSet_SSE41)) - { - intrinsic = NI_SSE41_Max; - break; - } - - uint64_t constVal = 0; - CorInfoType opJitType = simdBaseJitType; - var_types opType = simdBaseType; - genTreeOps fixupOp1 = GT_NONE; - genTreeOps fixupOp2 = GT_NONE; - - switch (simdBaseType) - { - case TYP_BYTE: - { - constVal = 0x8080808080808080; - fixupOp1 = GT_SUB; - fixupOp2 = GT_ADD; - simdBaseJitType = CORINFO_TYPE_UBYTE; - simdBaseType = TYP_UBYTE; - break; - } - - case TYP_USHORT: - { - constVal = 0x8000800080008000; - fixupOp1 = GT_ADD; - fixupOp2 = GT_SUB; - simdBaseJitType = CORINFO_TYPE_SHORT; - simdBaseType = TYP_SHORT; - break; - } - - default: - { - unreached(); - } - } - - assert(fixupOp1 != GT_NONE); - assert(fixupOp2 != GT_NONE); - assert(opJitType != simdBaseJitType); - assert(opType != simdBaseType); - - GenTreeVecCon* vecCon1 = gtNewVconNode(type); - - for (unsigned i = 0; i < (simdSize / 8); i++) - { - vecCon1->gtSimdVal.u64[i] = constVal; - } - - GenTree* vecCon2 = gtCloneExpr(vecCon1); - GenTree* vecCon3 = gtCloneExpr(vecCon2); - - // op1 = op1 - constVector - // -or- - // op1 = op1 + constVector - op1 = gtNewSimdBinOpNode(fixupOp1, type, op1, vecCon1, opJitType, simdSize); - - // op2 = op2 - constVector - // -or- - // op2 = op2 + constVector - op2 = gtNewSimdBinOpNode(fixupOp1, type, op2, vecCon2, opJitType, simdSize); - - // op1 = Max(op1, op2) - op1 = gtNewSimdMaxNativeNode(type, op1, op2, simdBaseJitType, simdSize); - - // result = op1 + constVector - // -or- - // result = op1 - constVector - return gtNewSimdBinOpNode(fixupOp2, type, op1, vecCon3, opJitType, simdSize); - } - - case TYP_INT: - case TYP_UINT: - { - if (compOpportunisticallyDependsOn(InstructionSet_SSE41)) - { - intrinsic = NI_SSE41_Max; - } - break; - } - - case TYP_LONG: - case TYP_ULONG: - { - if (compOpportunisticallyDependsOn(InstructionSet_AVX10v1)) - { - intrinsic = NI_AVX10v1_Max; - } - else if (compOpportunisticallyDependsOn(InstructionSet_AVX512F_VL)) - { - intrinsic = NI_AVX512F_VL_Max; - } - break; - } - - case TYP_FLOAT: - { - intrinsic = NI_SSE_Max; - break; - } - - case TYP_UBYTE: - case TYP_SHORT: - case TYP_DOUBLE: - { - intrinsic = NI_SSE2_Max; - break; - } - - default: - { - unreached(); - } - } - } -#elif defined(TARGET_ARM64) - if (!varTypeIsLong(simdBaseType)) - { - if (simdBaseType == TYP_DOUBLE) - { - intrinsic = (simdSize == 8) ? NI_AdvSimd_Arm64_MaxScalar : NI_AdvSimd_Arm64_Max; - } - else - { - intrinsic = NI_AdvSimd_Max; - } - } -#else -#error Unsupported platform -#endif // !TARGET_XARCH && !TARGET_ARM64 - - if (intrinsic != NI_Illegal) - { - return gtNewSimdHWIntrinsicNode(type, op1, op2, intrinsic, simdBaseJitType, simdSize); - } - - GenTree* op1Dup = fgMakeMultiUse(&op1); - GenTree* op2Dup = fgMakeMultiUse(&op2); - - // op1 = op1 > op2 - op1 = gtNewSimdCmpOpNode(GT_GT, type, op1, op2, simdBaseJitType, simdSize); - - // result = ConditionalSelect(op1, op1Dup, op2Dup) - return gtNewSimdCndSelNode(type, op1, op1Dup, op2Dup, simdBaseJitType, simdSize); -} - -GenTree* Compiler::gtNewSimdMinNode( - var_types type, GenTree* op1, GenTree* op2, CorInfoType simdBaseJitType, unsigned simdSize) -{ - assert(IsBaselineSimdIsaSupportedDebugOnly()); - - assert(varTypeIsSIMD(type)); - assert(getSIMDTypeForSize(simdSize) == type); - - assert(op1 != nullptr); - assert(op1->TypeIs(type)); - - assert(op2 != nullptr); - assert(op2->TypeIs(type)); - - var_types simdBaseType = JitType2PreciseVarType(simdBaseJitType); - assert(varTypeIsArithmetic(simdBaseType)); - -#if defined(TARGET_XARCH) - if (varTypeIsFloating(simdBaseType)) - { - GenTree* op1Dup1 = fgMakeMultiUse(&op1); - GenTree* op1Dup2 = gtCloneExpr(op1Dup1); - GenTree* op1Dup3 = gtCloneExpr(op1Dup2); - GenTree* op1Dup4 = gtCloneExpr(op1Dup3); - - GenTree* op2Dup1 = fgMakeMultiUse(&op2); - GenTree* op2Dup2 = gtCloneExpr(op2Dup1); - - GenTree* equalsMask = gtNewSimdCmpOpNode(GT_EQ, type, op1, op2, simdBaseJitType, simdSize); - GenTree* isNegativeMask = gtNewSimdIsNegativeNode(type, op1Dup1, simdBaseJitType, simdSize); - GenTree* isNaNMask = gtNewSimdIsNaNNode(type, op1Dup2, simdBaseJitType, simdSize); - GenTree* lessThanMask = gtNewSimdCmpOpNode(GT_LT, type, op1Dup3, op2Dup1, simdBaseJitType, simdSize); - - GenTree* mask = gtNewSimdBinOpNode(GT_AND, type, equalsMask, isNegativeMask, simdBaseJitType, simdSize); - mask = gtNewSimdBinOpNode(GT_OR, type, mask, isNaNMask, simdBaseJitType, simdSize); - mask = gtNewSimdBinOpNode(GT_OR, type, mask, lessThanMask, simdBaseJitType, simdSize); - - return gtNewSimdCndSelNode(type, mask, op1Dup4, op2Dup2, simdBaseJitType, simdSize); - } -#endif // TARGET_XARCH - - return gtNewSimdMinNativeNode(type, op1, op2, simdBaseJitType, simdSize); -} - -GenTree* Compiler::gtNewSimdMinNativeNode( - var_types type, GenTree* op1, GenTree* op2, CorInfoType simdBaseJitType, unsigned simdSize) -{ - assert(IsBaselineSimdIsaSupportedDebugOnly()); - - assert(varTypeIsSIMD(type)); - assert(getSIMDTypeForSize(simdSize) == type); - - assert(op1 != nullptr); - assert(op1->TypeIs(type)); - - assert(op2 != nullptr); - assert(op2->TypeIs(type)); - - var_types simdBaseType = JitType2PreciseVarType(simdBaseJitType); - assert(varTypeIsArithmetic(simdBaseType)); - - NamedIntrinsic intrinsic = NI_Illegal; - -#if defined(TARGET_XARCH) - if (simdSize == 32) - { - assert(compIsaSupportedDebugOnly(InstructionSet_AVX)); - - if (varTypeIsFloating(simdBaseType)) - { - intrinsic = NI_AVX_Min; - } - else - { - assert(compIsaSupportedDebugOnly(InstructionSet_AVX2)); - - if (!varTypeIsLong(simdBaseType)) - { - intrinsic = NI_AVX2_Min; - } - else if (compOpportunisticallyDependsOn(InstructionSet_AVX10v1)) - { - intrinsic = NI_AVX10v1_Min; - } - else if (compOpportunisticallyDependsOn(InstructionSet_AVX512F_VL)) - { - intrinsic = NI_AVX512F_VL_Min; - } - } - } - else if (simdSize == 64) - { - if (varTypeIsSmall(simdBaseType)) - { - assert(compIsaSupportedDebugOnly(InstructionSet_AVX512BW)); - intrinsic = NI_AVX512BW_Min; - } - else - { - assert(compIsaSupportedDebugOnly(InstructionSet_AVX512F)); - intrinsic = NI_AVX512F_Min; - } - } - else - { - switch (simdBaseType) - { - case TYP_BYTE: - case TYP_USHORT: - { - if (compOpportunisticallyDependsOn(InstructionSet_SSE41)) - { - intrinsic = NI_SSE41_Min; - break; - } - - GenTree* constVal = nullptr; - CorInfoType opJitType = simdBaseJitType; - var_types opType = simdBaseType; - genTreeOps fixupOp1 = GT_NONE; - genTreeOps fixupOp2 = GT_NONE; - - switch (simdBaseType) - { - case TYP_BYTE: - { - constVal = gtNewIconNode(0x80808080); - fixupOp1 = GT_SUB; - fixupOp2 = GT_ADD; - simdBaseJitType = CORINFO_TYPE_UBYTE; - simdBaseType = TYP_UBYTE; - break; - } - - case TYP_USHORT: - { - constVal = gtNewIconNode(0x80008000); - fixupOp1 = GT_ADD; - fixupOp2 = GT_SUB; - simdBaseJitType = CORINFO_TYPE_SHORT; - simdBaseType = TYP_SHORT; - break; - } - - default: - { - unreached(); - } - } - - assert(constVal != nullptr); - assert(fixupOp1 != GT_NONE); - assert(fixupOp2 != GT_NONE); - assert(opJitType != simdBaseJitType); - assert(opType != simdBaseType); - - GenTree* constVector = gtNewSimdCreateBroadcastNode(type, constVal, CORINFO_TYPE_INT, simdSize); - - GenTree* constVectorDup1 = fgMakeMultiUse(&constVector); - GenTree* constVectorDup2 = gtCloneExpr(constVectorDup1); - - // op1 = op1 - constVector - // -or- - // op1 = op1 + constVector - op1 = gtNewSimdBinOpNode(fixupOp1, type, op1, constVector, opJitType, simdSize); - - // op2 = op2 - constVectorDup1 - // -or- - // op2 = op2 + constVectorDup1 - op2 = gtNewSimdBinOpNode(fixupOp1, type, op2, constVectorDup1, opJitType, simdSize); - - // op1 = Min(op1, op2) - op1 = gtNewSimdMinNativeNode(type, op1, op2, simdBaseJitType, simdSize); - - // result = op1 + constVectorDup2 - // -or- - // result = op1 - constVectorDup2 - return gtNewSimdBinOpNode(fixupOp2, type, op1, constVectorDup2, opJitType, simdSize); - } - - case TYP_INT: - case TYP_UINT: - { - if (compOpportunisticallyDependsOn(InstructionSet_SSE41)) - { - intrinsic = NI_SSE41_Min; - } - break; - } - - case TYP_LONG: - case TYP_ULONG: - { - if (compOpportunisticallyDependsOn(InstructionSet_AVX10v1)) - { - intrinsic = NI_AVX10v1_Min; - } - else if (compOpportunisticallyDependsOn(InstructionSet_AVX512F_VL)) - { - intrinsic = NI_AVX512F_VL_Min; - } - break; - } - - case TYP_FLOAT: - { - intrinsic = NI_SSE_Min; - break; - } - - case TYP_UBYTE: - case TYP_SHORT: - case TYP_DOUBLE: - { - intrinsic = NI_SSE2_Min; - break; - } - - default: - { - unreached(); - } - } - } -#elif defined(TARGET_ARM64) - if (!varTypeIsLong(simdBaseType)) - { - if (simdBaseType == TYP_DOUBLE) - { - intrinsic = (simdSize == 8) ? NI_AdvSimd_Arm64_MinScalar : NI_AdvSimd_Arm64_Min; - } - else - { - intrinsic = NI_AdvSimd_Min; - } - } -#else -#error Unsupported platform -#endif // !TARGET_XARCH && !TARGET_ARM64 - - if (intrinsic != NI_Illegal) - { - return gtNewSimdHWIntrinsicNode(type, op1, op2, intrinsic, simdBaseJitType, simdSize); - } - - GenTree* op1Dup = fgMakeMultiUse(&op1); - GenTree* op2Dup = fgMakeMultiUse(&op2); - - // op1 = op1 < op2 - op1 = gtNewSimdCmpOpNode(GT_LT, type, op1, op2, simdBaseJitType, simdSize); - - // result = ConditionalSelect(op1, op1Dup, op2Dup) - return gtNewSimdCndSelNode(type, op1, op1Dup, op2Dup, simdBaseJitType, simdSize); -} - -GenTree* Compiler::gtNewSimdNarrowNode( - var_types type, GenTree* op1, GenTree* op2, CorInfoType simdBaseJitType, unsigned simdSize) -{ - assert(IsBaselineSimdIsaSupportedDebugOnly()); - - assert(varTypeIsSIMD(type)); - assert(getSIMDTypeForSize(simdSize) == type); - - assert(op1 != nullptr); - assert(op1->TypeIs(type)); - - assert(op2 != nullptr); - assert(op2->TypeIs(type)); - - var_types simdBaseType = JitType2PreciseVarType(simdBaseJitType); - assert(varTypeIsArithmetic(simdBaseType) && !varTypeIsLong(simdBaseType)); - - GenTree* tmp1; - GenTree* tmp2; - - bool isV512Supported = false; - -#if defined(TARGET_XARCH) - GenTree* tmp3; - GenTree* tmp4; - if (compIsEvexOpportunisticallySupported(isV512Supported)) - { - // This is the same in principle to the other comments below, however due to - // code formatting, its too long to reasonably display here. - - assert((simdSize == 16) || (simdSize == 32) || (simdSize == 64)); - var_types tmpSimdType = (simdSize == 64) ? TYP_SIMD32 : TYP_SIMD16; - - NamedIntrinsic intrinsicId; - CorInfoType opBaseJitType; - - switch (simdBaseType) - { - case TYP_BYTE: - { - if (simdSize == 64) - { - intrinsicId = NI_AVX512BW_ConvertToVector256SByte; - } - else - { - intrinsicId = - !isV512Supported ? NI_AVX10v1_ConvertToVector128SByte : NI_AVX512BW_VL_ConvertToVector128SByte; - } - - opBaseJitType = CORINFO_TYPE_SHORT; - break; - } - - case TYP_UBYTE: - { - if (simdSize == 64) - { - intrinsicId = NI_AVX512BW_ConvertToVector256Byte; - } - else - { - intrinsicId = - !isV512Supported ? NI_AVX10v1_ConvertToVector128Byte : NI_AVX512BW_VL_ConvertToVector128Byte; - } - - opBaseJitType = CORINFO_TYPE_USHORT; - break; - } - - case TYP_SHORT: - { - if (simdSize == 64) - { - intrinsicId = NI_AVX512F_ConvertToVector256Int16; - } - else - { - intrinsicId = - !isV512Supported ? NI_AVX10v1_ConvertToVector128Int16 : NI_AVX512F_VL_ConvertToVector128Int16; - } - - opBaseJitType = CORINFO_TYPE_INT; - break; - } - - case TYP_USHORT: - { - if (simdSize == 64) - { - intrinsicId = NI_AVX512F_ConvertToVector256UInt16; - } - else - { - intrinsicId = - !isV512Supported ? NI_AVX10v1_ConvertToVector128UInt16 : NI_AVX512F_VL_ConvertToVector128UInt16; - } - - opBaseJitType = CORINFO_TYPE_UINT; - break; - } - - case TYP_INT: - { - if (simdSize == 64) - { - intrinsicId = NI_AVX512F_ConvertToVector256Int32; - } - else - { - intrinsicId = - !isV512Supported ? NI_AVX10v1_ConvertToVector128Int32 : NI_AVX512F_VL_ConvertToVector128Int32; - } - - opBaseJitType = CORINFO_TYPE_LONG; - break; - } - - case TYP_UINT: - { - if (simdSize == 64) - { - intrinsicId = NI_AVX512F_ConvertToVector256UInt32; - } - else - { - intrinsicId = - !isV512Supported ? NI_AVX10v1_ConvertToVector128UInt32 : NI_AVX512F_VL_ConvertToVector128UInt32; - } - - opBaseJitType = CORINFO_TYPE_ULONG; - break; - } - - case TYP_FLOAT: - { - if (simdSize == 64) - { - intrinsicId = NI_AVX512F_ConvertToVector256Single; - } - else if (simdSize == 32) - { - intrinsicId = NI_AVX_ConvertToVector128Single; - } - else - { - intrinsicId = NI_SSE2_ConvertToVector128Single; - } - - opBaseJitType = CORINFO_TYPE_DOUBLE; - break; - } - - default: - { - unreached(); - } - } - - tmp1 = gtNewSimdHWIntrinsicNode(tmpSimdType, op1, intrinsicId, opBaseJitType, simdSize); - tmp2 = gtNewSimdHWIntrinsicNode(tmpSimdType, op2, intrinsicId, opBaseJitType, simdSize); - - if (simdSize == 16) - { - return gtNewSimdHWIntrinsicNode(type, tmp1, tmp2, NI_SSE_MoveLowToHigh, CORINFO_TYPE_FLOAT, simdSize); - } - - intrinsicId = (simdSize == 64) ? NI_Vector256_ToVector512Unsafe : NI_Vector128_ToVector256Unsafe; - - tmp1 = gtNewSimdHWIntrinsicNode(type, tmp1, intrinsicId, simdBaseJitType, simdSize / 2); - return gtNewSimdWithUpperNode(type, tmp1, tmp2, simdBaseJitType, simdSize); - } - else if (simdSize == 32) - { - assert(compIsaSupportedDebugOnly(InstructionSet_AVX)); - - switch (simdBaseType) - { - case TYP_BYTE: - case TYP_UBYTE: - { - assert(compIsaSupportedDebugOnly(InstructionSet_AVX2)); - - // This is the same in principle to the other comments below, however due to - // code formatting, its too long to reasonably display here. - GenTreeVecCon* vecCon1 = gtNewVconNode(type); - - for (unsigned i = 0; i < (simdSize / 8); i++) - { - vecCon1->gtSimdVal.u64[i] = 0x00FF00FF00FF00FF; - } - - GenTree* vecCon2 = gtCloneExpr(vecCon1); - - tmp1 = gtNewSimdBinOpNode(GT_AND, type, op1, vecCon1, simdBaseJitType, simdSize); - tmp2 = gtNewSimdBinOpNode(GT_AND, type, op2, vecCon2, simdBaseJitType, simdSize); - tmp3 = gtNewSimdHWIntrinsicNode(type, tmp1, tmp2, NI_AVX2_PackUnsignedSaturate, CORINFO_TYPE_UBYTE, - simdSize); - - CorInfoType permuteBaseJitType = (simdBaseType == TYP_BYTE) ? CORINFO_TYPE_LONG : CORINFO_TYPE_ULONG; - return gtNewSimdHWIntrinsicNode(type, tmp3, gtNewIconNode(SHUFFLE_WYZX), NI_AVX2_Permute4x64, - permuteBaseJitType, simdSize); - } - - case TYP_SHORT: - case TYP_USHORT: - { - assert(compIsaSupportedDebugOnly(InstructionSet_AVX2)); - - // op1 = Elements 0L, 0U, 1L, 1U, 2L, 2U, 3L, 3U | 4L, 4U, 5L, 5U, 6L, 6U, 7L, 7U - // op2 = Elements 8L, 8U, 9L, 9U, AL, AU, BL, BU | CL, CU, DL, DU, EL, EU, FL, FU - // - // tmp2 = Elements 0L, --, 1L, --, 2L, --, 3L, -- | 4L, --, 5L, --, 6L, --, 7L, -- - // tmp3 = Elements 8L, --, 9L, --, AL, --, BL, -- | CL, --, DL, --, EL, --, FL, -- - // tmp4 = Elements 0L, 1L, 2L, 3L, 8L, 9L, AL, BL | 4L, 5L, 6L, 7L, CL, DL, EL, FL - // return Elements 0L, 1L, 2L, 3L, 4L, 5L, 6L, 7L | 8L, 9L, AL, BL, CL, DL, EL, FL - // - // var vcns = Vector256.Create(0x0000FFFF).AsInt16(); - // var tmp1 = Avx2.And(op1.AsInt16(), vcns); - // var tmp2 = Avx2.And(op2.AsInt16(), vcns); - // var tmp3 = Avx2.PackUnsignedSaturate(tmp1, tmp2); - // return Avx2.Permute4x64(tmp3.AsUInt64(), SHUFFLE_WYZX).As(); - - GenTreeVecCon* vecCon1 = gtNewVconNode(type); - - for (unsigned i = 0; i < (simdSize / 8); i++) - { - vecCon1->gtSimdVal.u64[i] = 0x0000FFFF0000FFFF; - } - - GenTree* vecCon2 = gtCloneExpr(vecCon1); - - tmp1 = gtNewSimdBinOpNode(GT_AND, type, op1, vecCon1, simdBaseJitType, simdSize); - tmp2 = gtNewSimdBinOpNode(GT_AND, type, op2, vecCon2, simdBaseJitType, simdSize); - tmp3 = gtNewSimdHWIntrinsicNode(type, tmp1, tmp2, NI_AVX2_PackUnsignedSaturate, CORINFO_TYPE_USHORT, - simdSize); - - CorInfoType permuteBaseJitType = (simdBaseType == TYP_BYTE) ? CORINFO_TYPE_LONG : CORINFO_TYPE_ULONG; - return gtNewSimdHWIntrinsicNode(type, tmp3, gtNewIconNode(SHUFFLE_WYZX), NI_AVX2_Permute4x64, - permuteBaseJitType, simdSize); - } - - case TYP_INT: - case TYP_UINT: - { - assert(compIsaSupportedDebugOnly(InstructionSet_AVX2)); - - // op1 = Elements 0, 1 | 2, 3; 0L, 0U, 1L, 1U | 2L, 2U, 3L, 3U - // op2 = Elements 4, 5 | 6, 7; 4L, 4U, 5L, 5U | 6L, 6U, 7L, 7U - // - // tmp1 = Elements 0L, 4L, 0U, 4U | 2L, 6L, 2U, 6U - // tmp2 = Elements 1L, 5L, 1U, 5U | 3L, 7L, 3U, 7U - // tmp3 = Elements 0L, 1L, 4L, 5L | 2L, 3L, 6L, 7L - // return Elements 0L, 1L, 2L, 3L | 4L, 5L, 6L, 7L - // - // var tmp1 = Avx2.UnpackLow(op1, op2); - // var tmp2 = Avx2.UnpackHigh(op1, op2); - // var tmp3 = Avx2.UnpackLow(tmp1, tmp2); - // return Avx2.Permute4x64(tmp3.AsUInt64(), SHUFFLE_WYZX).AsUInt32(); - - CorInfoType opBaseJitType = (simdBaseType == TYP_INT) ? CORINFO_TYPE_LONG : CORINFO_TYPE_ULONG; - - GenTree* op1Dup = fgMakeMultiUse(&op1); - GenTree* op2Dup = fgMakeMultiUse(&op2); - - tmp1 = gtNewSimdHWIntrinsicNode(type, op1, op2, NI_AVX2_UnpackLow, simdBaseJitType, simdSize); - tmp2 = gtNewSimdHWIntrinsicNode(type, op1Dup, op2Dup, NI_AVX2_UnpackHigh, simdBaseJitType, simdSize); - tmp3 = gtNewSimdHWIntrinsicNode(type, tmp1, tmp2, NI_AVX2_UnpackLow, simdBaseJitType, simdSize); - - return gtNewSimdHWIntrinsicNode(type, tmp3, gtNewIconNode(SHUFFLE_WYZX), NI_AVX2_Permute4x64, - opBaseJitType, simdSize); - } - - case TYP_FLOAT: - { - // op1 = Elements 0, 1 | 2, 3 - // op2 = Elements 4, 5 | 6, 7 - // - // tmp1 = Elements 0, 1, 2, 3 | -, -, -, - - // tmp1 = Elements 4, 5, 6, 7 - // return Elements 0, 1, 2, 3 | 4, 5, 6, 7 - // - // var tmp1 = Avx.ConvertToVector128Single(op1).ToVector256Unsafe(); - // var tmp2 = Avx.ConvertToVector128Single(op2); - // return tmp1.WithUpper(tmp2); - - CorInfoType opBaseJitType = CORINFO_TYPE_DOUBLE; - - tmp1 = - gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, NI_AVX_ConvertToVector128Single, opBaseJitType, simdSize); - tmp2 = - gtNewSimdHWIntrinsicNode(TYP_SIMD16, op2, NI_AVX_ConvertToVector128Single, opBaseJitType, simdSize); - - tmp1 = gtNewSimdHWIntrinsicNode(type, tmp1, NI_Vector128_ToVector256Unsafe, simdBaseJitType, 16); - return gtNewSimdWithUpperNode(type, tmp1, tmp2, simdBaseJitType, simdSize); - } - - default: - { - unreached(); - } - } - } - else - { - assert(simdSize == 16); - - switch (simdBaseType) - { - case TYP_BYTE: - case TYP_UBYTE: - { - // op1 = Elements 0, 1, 2, 3, 4, 5, 6, 7; 0L, 0U, 1L, 1U, 2L, 2U, 3L, 3U, 4L, 4U, 5L, 5U, 6L, 6U, 7L, 7U - // op2 = Elements 8, 9, A, B, C, D, E, F; 8L, 8U, 9L, 9U, AL, AU, BL, BU, CL, CU, DL, DU, EL, EU, FL, FU - // - // tmp2 = Elements 0L, --, 1L, --, 2L, --, 3L, --, 4L, --, 5L, --, 6L, --, 7L, -- - // tmp3 = Elements 8L, --, 9L, --, AL, --, BL, --, CL, --, DL, --, EL, --, FL, -- - // return Elements 0L, 1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L, AL, BL, CL, DL, EL, FL - // - // var vcns = Vector128.Create((ushort)(0x00FF)).AsSByte(); - // var tmp1 = Sse2.And(op1.AsSByte(), vcns); - // var tmp2 = Sse2.And(op2.AsSByte(), vcns); - // return Sse2.PackUnsignedSaturate(tmp1, tmp2).As(); - - GenTreeVecCon* vecCon1 = gtNewVconNode(type); - - for (unsigned i = 0; i < (simdSize / 8); i++) - { - vecCon1->gtSimdVal.u64[i] = 0x00FF00FF00FF00FF; - } - - GenTree* vecCon2 = gtCloneExpr(vecCon1); - - tmp1 = gtNewSimdBinOpNode(GT_AND, type, op1, vecCon1, simdBaseJitType, simdSize); - tmp2 = gtNewSimdBinOpNode(GT_AND, type, op2, vecCon2, simdBaseJitType, simdSize); - - return gtNewSimdHWIntrinsicNode(type, tmp1, tmp2, NI_SSE2_PackUnsignedSaturate, CORINFO_TYPE_UBYTE, - simdSize); - } - - case TYP_SHORT: - case TYP_USHORT: - { - // op1 = Elements 0, 1, 2, 3; 0L, 0U, 1L, 1U, 2L, 2U, 3L, 3U - // op2 = Elements 4, 5, 6, 7; 4L, 4U, 5L, 5U, 6L, 6U, 7L, 7U - // - // ... - if (compOpportunisticallyDependsOn(InstructionSet_SSE41)) - { - // ... - // - // tmp2 = Elements 0L, --, 1L, --, 2L, --, 3L, -- - // tmp3 = Elements 4L, --, 5L, --, 6L, --, 7L, -- - // return Elements 0L, 1L, 2L, 3L, 4L, 5L, 6L, 7L - // - // var vcns = Vector128.Create(0x0000FFFF).AsInt16(); - // var tmp1 = Sse2.And(op1.AsInt16(), vcns); - // var tmp2 = Sse2.And(op2.AsInt16(), vcns); - // return Sse2.PackUnsignedSaturate(tmp1, tmp2).As(); - - GenTreeVecCon* vecCon1 = gtNewVconNode(type); - - for (unsigned i = 0; i < (simdSize / 8); i++) - { - vecCon1->gtSimdVal.u64[i] = 0x0000FFFF0000FFFF; - } - - GenTree* vecCon2 = gtCloneExpr(vecCon1); - - tmp1 = gtNewSimdBinOpNode(GT_AND, type, op1, vecCon1, simdBaseJitType, simdSize); - tmp2 = gtNewSimdBinOpNode(GT_AND, type, op2, vecCon2, simdBaseJitType, simdSize); - - return gtNewSimdHWIntrinsicNode(type, tmp1, tmp2, NI_SSE41_PackUnsignedSaturate, - CORINFO_TYPE_USHORT, simdSize); - } - else - { - // ... - // - // tmp1 = Elements 0L, 4L, 0U, 4U, 1L, 5L, 1U, 5U - // tmp2 = Elements 2L, 6L, 2U, 6U, 3L, 7L, 3U, 7U - // tmp3 = Elements 0L, 2L, 4L, 6L, 0U, 2U, 4U, 6U - // tmp4 = Elements 1L, 3L, 5L, 7L, 1U, 3U, 5U, 7U - // return Elements 0L, 1L, 2L, 3L, 4L, 5L, 6L, 7L - // - // var tmp1 = Sse2.UnpackLow(op1.AsUInt16(), op2.AsUInt16()); - // var tmp2 = Sse2.UnpackHigh(op1.AsUInt16(), op2.AsUInt16()); - // var tmp3 = Sse2.UnpackLow(tmp1, tmp2); - // var tmp4 = Sse2.UnpackHigh(tmp1, tmp2); - // return Sse2.UnpackLow(tmp3, tmp4).As(); - - GenTree* op1Dup = fgMakeMultiUse(&op1); - GenTree* op2Dup = fgMakeMultiUse(&op2); - - tmp1 = gtNewSimdHWIntrinsicNode(type, op1, op2, NI_SSE2_UnpackLow, simdBaseJitType, simdSize); - tmp2 = - gtNewSimdHWIntrinsicNode(type, op1Dup, op2Dup, NI_SSE2_UnpackHigh, simdBaseJitType, simdSize); - - GenTree* tmp1Dup = fgMakeMultiUse(&tmp1); - GenTree* tmp2Dup = fgMakeMultiUse(&tmp2); - - tmp3 = gtNewSimdHWIntrinsicNode(type, tmp1, tmp2, NI_SSE2_UnpackLow, simdBaseJitType, simdSize); - tmp4 = - gtNewSimdHWIntrinsicNode(type, tmp1Dup, tmp2Dup, NI_SSE2_UnpackHigh, simdBaseJitType, simdSize); - - return gtNewSimdHWIntrinsicNode(type, tmp3, tmp4, NI_SSE2_UnpackLow, simdBaseJitType, simdSize); - } - } - - case TYP_INT: - case TYP_UINT: - { - // op1 = Elements 0, 1; 0L, 0U, 1L, 1U - // op2 = Elements 2, 3; 2L, 2U, 3L, 3U - // - // tmp1 = Elements 0L, 2L, 0U, 2U - // tmp2 = Elements 1L, 3L, 1U, 3U - // return Elements 0L, 1L, 2L, 3L - // - // var tmp1 = Sse2.UnpackLow(op1.AsUInt32(), op2.AsUInt32()); - // var tmp2 = Sse2.UnpackHigh(op1.AsUInt32(), op2.AsUInt32()); - // return Sse2.UnpackLow(tmp1, tmp2).As(); - - GenTree* op1Dup = fgMakeMultiUse(&op1); - GenTree* op2Dup = fgMakeMultiUse(&op2); - - tmp1 = gtNewSimdHWIntrinsicNode(type, op1, op2, NI_SSE2_UnpackLow, simdBaseJitType, simdSize); - tmp2 = gtNewSimdHWIntrinsicNode(type, op1Dup, op2Dup, NI_SSE2_UnpackHigh, simdBaseJitType, simdSize); - - return gtNewSimdHWIntrinsicNode(type, tmp1, tmp2, NI_SSE2_UnpackLow, simdBaseJitType, simdSize); - } - - case TYP_FLOAT: - { - // op1 = Elements 0, 1 - // op2 = Elements 2, 3 - // - // tmp1 = Elements 0, 1, -, - - // tmp1 = Elements 2, 3, -, - - // return Elements 0, 1, 2, 3 - // - // var tmp1 = Sse2.ConvertToVector128Single(op1); - // var tmp2 = Sse2.ConvertToVector128Single(op2); - // return Sse.MoveLowToHigh(tmp1, tmp2); - - CorInfoType opBaseJitType = CORINFO_TYPE_DOUBLE; - - tmp1 = gtNewSimdHWIntrinsicNode(type, op1, NI_SSE2_ConvertToVector128Single, opBaseJitType, simdSize); - tmp2 = gtNewSimdHWIntrinsicNode(type, op2, NI_SSE2_ConvertToVector128Single, opBaseJitType, simdSize); - - return gtNewSimdHWIntrinsicNode(type, tmp1, tmp2, NI_SSE_MoveLowToHigh, simdBaseJitType, simdSize); - } - - default: - { - unreached(); - } - } - } -#elif defined(TARGET_ARM64) - if (simdSize == 16) - { - if (varTypeIsFloating(simdBaseType)) - { - // var tmp1 = AdvSimd.Arm64.ConvertToSingleLower(op1); - // return AdvSimd.Arm64.ConvertToSingleUpper(tmp1, op2); - - tmp1 = gtNewSimdHWIntrinsicNode(TYP_SIMD8, op1, NI_AdvSimd_Arm64_ConvertToSingleLower, simdBaseJitType, 8); - return gtNewSimdHWIntrinsicNode(type, tmp1, op2, NI_AdvSimd_Arm64_ConvertToSingleUpper, simdBaseJitType, - simdSize); - } - else - { - // var tmp1 = AdvSimd.ExtractNarrowingLower(op1); - // return AdvSimd.ExtractNarrowingUpper(tmp1, op2); - - tmp1 = gtNewSimdHWIntrinsicNode(TYP_SIMD8, op1, NI_AdvSimd_ExtractNarrowingLower, simdBaseJitType, 8); - return gtNewSimdHWIntrinsicNode(type, tmp1, op2, NI_AdvSimd_ExtractNarrowingUpper, simdBaseJitType, - simdSize); - } - } - else if (varTypeIsFloating(simdBaseType)) - { - // var tmp1 = op1.ToVector128Unsafe(); - // var tmp2 = AdvSimd.InsertScalar(tmp1, op2); - // return AdvSimd.Arm64.ConvertToSingleLower(tmp2); - - CorInfoType tmp2BaseJitType = CORINFO_TYPE_DOUBLE; - - tmp1 = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, NI_Vector64_ToVector128Unsafe, simdBaseJitType, simdSize); - tmp2 = gtNewSimdWithUpperNode(TYP_SIMD16, tmp1, op2, tmp2BaseJitType, 16); - - return gtNewSimdHWIntrinsicNode(type, tmp2, NI_AdvSimd_Arm64_ConvertToSingleLower, simdBaseJitType, simdSize); - } - else - { - // var tmp1 = op1.ToVector128Unsafe(); - // var tmp2 = AdvSimd.InsertScalar(tmp1.AsUInt64(), 1, op2.AsUInt64()).As(); - signed integer use int64, - // unsigned integer use uint64 - // return AdvSimd.ExtractNarrowingLower(tmp2); - - CorInfoType tmp2BaseJitType = varTypeIsSigned(simdBaseType) ? CORINFO_TYPE_LONG : CORINFO_TYPE_ULONG; - - tmp1 = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, NI_Vector64_ToVector128Unsafe, simdBaseJitType, simdSize); - tmp2 = gtNewSimdWithUpperNode(TYP_SIMD16, tmp1, op2, tmp2BaseJitType, 16); - - return gtNewSimdHWIntrinsicNode(type, tmp2, NI_AdvSimd_ExtractNarrowingLower, simdBaseJitType, simdSize); - } -#else -#error Unsupported platform -#endif // !TARGET_XARCH && !TARGET_ARM64 -} - -//------------------------------------------------------------------------ -// gtNewSimdRoundNode: Creates a new simd Round node -// -// Arguments: -// type -- The type of the node -// op1 -- The node to round -// simdBaseJitType -- the base jit type of the node -// simdSize -- the simd size of the node -// -// Return Value: -// The round node -// -GenTree* Compiler::gtNewSimdRoundNode(var_types type, GenTree* op1, CorInfoType simdBaseJitType, unsigned simdSize) -{ - assert(IsBaselineSimdIsaSupportedDebugOnly()); - - assert(varTypeIsSIMD(type)); - assert(getSIMDTypeForSize(simdSize) == type); - - assert(op1 != nullptr); - assert(op1->TypeIs(type)); - - var_types simdBaseType = JitType2PreciseVarType(simdBaseJitType); - assert(varTypeIsFloating(simdBaseType)); - - NamedIntrinsic intrinsic = NI_Illegal; - -#if defined(TARGET_XARCH) - if (simdSize == 32) - { - assert(compIsaSupportedDebugOnly(InstructionSet_AVX)); - intrinsic = NI_AVX_RoundToNearestInteger; - } - else if (simdSize == 64) - { - assert(compIsaSupportedDebugOnly(InstructionSet_AVX512F)); - GenTree* op2 = gtNewIconNode(static_cast(FloatRoundingMode::ToNearestInteger)); - return gtNewSimdHWIntrinsicNode(type, op1, op2, NI_AVX512F_RoundScale, simdBaseJitType, simdSize); - } - else - { - assert(compIsaSupportedDebugOnly(InstructionSet_SSE41)); - intrinsic = NI_SSE41_RoundToNearestInteger; - } -#elif defined(TARGET_ARM64) - if (simdBaseType == TYP_DOUBLE) - { - intrinsic = (simdSize == 8) ? NI_AdvSimd_RoundToNearestScalar : NI_AdvSimd_Arm64_RoundToNearest; - } - else - { - intrinsic = NI_AdvSimd_RoundToNearest; - } -#else -#error Unsupported platform -#endif // !TARGET_XARCH && !TARGET_ARM64 - - assert(intrinsic != NI_Illegal); - return gtNewSimdHWIntrinsicNode(type, op1, intrinsic, simdBaseJitType, simdSize); -} - -GenTree* Compiler::gtNewSimdShuffleNode( - var_types type, GenTree* op1, GenTree* op2, CorInfoType simdBaseJitType, unsigned simdSize) -{ - assert(IsBaselineSimdIsaSupportedDebugOnly()); - - assert(varTypeIsSIMD(type)); - assert(getSIMDTypeForSize(simdSize) == type); - - assert(op1 != nullptr); - assert(op1->TypeIs(type)); - - assert(op2 != nullptr); - assert(op2->TypeIs(type)); - assert(op2->IsCnsVec()); - - var_types simdBaseType = JitType2PreciseVarType(simdBaseJitType); - assert(varTypeIsArithmetic(simdBaseType)); - - if (op2->IsVectorAllBitsSet()) - { - // AllBitsSet represents indices that are always "out of range" which means zero should be - // selected for every element. We can special-case this down to just returning a zero node - return gtNewZeroConNode(type); - } - - if (op2->IsVectorZero()) - { - // TODO-XARCH-CQ: Zero represents indices that select the first element of op1 each time. We can simplify - // this down to basically a broadcast equivalent. - } - - GenTree* retNode = nullptr; - GenTreeIntConCommon* cnsNode = nullptr; - - size_t elementSize = genTypeSize(simdBaseType); - size_t elementCount = simdSize / elementSize; - -#if defined(TARGET_XARCH) - uint8_t control = 0; - bool crossLane = false; - bool needsZero = varTypeIsSmall(simdBaseType) && (simdSize <= 16); - uint64_t value = 0; - simd_t vecCns = {}; - simd_t mskCns = {}; - - for (size_t index = 0; index < elementCount; index++) - { - value = op2->GetIntegralVectorConstElement(index, simdBaseType); - - if (value < elementCount) - { - if (simdSize == 32) - { - // Most of the 256-bit shuffle/permute instructions operate as if - // the inputs were 2x 128-bit values. If the selected indices cross - // the respective 128-bit "lane" we may need to specialize the codegen - - if (index < (elementCount / 2)) - { - crossLane |= (value >= (elementCount / 2)); - } - else - { - crossLane |= (value < (elementCount / 2)); - } - } - - // Setting the control for byte/sbyte and short/ushort is unnecessary - // and will actually compute an incorrect control word. But it simplifies - // the overall logic needed here and will remain unused. - - control |= (value << (index * (elementCount / 2))); - - // When Ssse3 is supported, we may need vecCns to accurately select the relevant - // bytes if some index is outside the valid range. Since x86/x64 is little-endian - // we can simplify this down to a for loop that scales the value and selects count - // sequential bytes. - - for (uint32_t i = 0; i < elementSize; i++) - { - vecCns.u8[(index * elementSize) + i] = (uint8_t)((value * elementSize) + i); - - // When Ssse3 is not supported, we need to adjust the constant to be AllBitsSet - // so that we can emit a ConditionalSelect(op2, retNode, zeroNode). - - mskCns.u8[(index * elementSize) + i] = 0xFF; - } - } - else - { - needsZero = true; - - // When Ssse3 is supported, we may need vecCns to accurately select the relevant - // bytes if some index is outside the valid range. We can do this by just zeroing - // out each byte in the element. This only requires the most significant bit to be - // set, but we use 0xFF instead since that will be the equivalent of AllBitsSet - - for (uint32_t i = 0; i < elementSize; i++) - { - vecCns.u8[(index * elementSize) + i] = 0xFF; - - // When Ssse3 is not supported, we need to adjust the constant to be Zero - // so that we can emit a ConditionalSelect(op2, retNode, zeroNode). - - mskCns.u8[(index * elementSize) + i] = 0x00; - } - } - } - - if (simdSize == 32) - { - assert(compIsaSupportedDebugOnly(InstructionSet_AVX2)); - bool isV512Supported = false; - if ((varTypeIsByte(simdBaseType) && - !compIsEvexOpportunisticallySupported(isV512Supported, InstructionSet_AVX512VBMI_VL)) || - (varTypeIsShort(simdBaseType) && - !compIsEvexOpportunisticallySupported(isV512Supported, InstructionSet_AVX512BW_VL))) - { - if (crossLane) - { - // TODO-XARCH-CQ: We should emulate cross-lane shuffling for byte/sbyte and short/ushort - unreached(); - } - - // If we aren't crossing lanes, then we can decompose the byte/sbyte - // and short/ushort operations into 2x 128-bit operations - - // We want to build what is essentially the following managed code: - // var op1Lower = op1.GetLower(); - // op1Lower = Ssse3.Shuffle(op1Lower, Vector128.Create(...)); - // - // var op1Upper = op1.GetUpper(); - // op1Upper = Ssse3.Shuffle(op1Upper, Vector128.Create(...)); - // - // return Vector256.Create(op1Lower, op1Upper); - - simdBaseJitType = varTypeIsUnsigned(simdBaseType) ? CORINFO_TYPE_UBYTE : CORINFO_TYPE_BYTE; - - GenTree* op1Dup = fgMakeMultiUse(&op1); - GenTree* op1Lower = gtNewSimdGetLowerNode(TYP_SIMD16, op1, simdBaseJitType, simdSize); - - op2 = gtNewVconNode(TYP_SIMD16); - op2->AsVecCon()->gtSimd16Val = vecCns.v128[0]; - - op1Lower = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1Lower, op2, NI_SSSE3_Shuffle, simdBaseJitType, 16); - - GenTree* op1Upper = gtNewSimdGetUpperNode(TYP_SIMD16, op1Dup, simdBaseJitType, simdSize); - - op2 = gtNewVconNode(TYP_SIMD16); - op2->AsVecCon()->gtSimd16Val = vecCns.v128[1]; - - op1Upper = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1Upper, op2, NI_SSSE3_Shuffle, simdBaseJitType, 16); - - return gtNewSimdWithUpperNode(type, op1Lower, op1Upper, simdBaseJitType, simdSize); - } - - if (elementSize == 4) - { - for (uint32_t i = 0; i < elementCount; i++) - { - vecCns.u32[i] = (uint8_t)(vecCns.u8[i * elementSize] / elementSize); - } - - op2 = gtNewVconNode(type); - op2->AsVecCon()->gtSimdVal = vecCns; - - // swap the operands to match the encoding requirements - retNode = gtNewSimdHWIntrinsicNode(type, op2, op1, NI_AVX2_PermuteVar8x32, simdBaseJitType, simdSize); - } - else if (elementSize == 2) - { - assert(canUseEvexEncodingDebugOnly()); - for (uint32_t i = 0; i < elementCount; i++) - { - vecCns.u16[i] = (uint8_t)(vecCns.u8[i * elementSize] / elementSize); - } - - op2 = gtNewVconNode(type); - op2->AsVecCon()->gtSimdVal = vecCns; - - // swap the operands to match the encoding requirements - retNode = !isV512Supported ? gtNewSimdHWIntrinsicNode(type, op2, op1, NI_AVX10v1_PermuteVar16x16, - simdBaseJitType, simdSize) - : gtNewSimdHWIntrinsicNode(type, op2, op1, NI_AVX512BW_VL_PermuteVar16x16, - simdBaseJitType, simdSize); - } - else if (elementSize == 1) - { - assert(IsAvx10OrIsaSupportedDebugOnly(InstructionSet_AVX512VBMI_VL)); - op2 = gtNewVconNode(type); - op2->AsVecCon()->gtSimdVal = vecCns; - - // swap the operands to match the encoding requirements - retNode = !isV512Supported ? gtNewSimdHWIntrinsicNode(type, op2, op1, NI_AVX10v1_PermuteVar32x8, - simdBaseJitType, simdSize) - : gtNewSimdHWIntrinsicNode(type, op2, op1, NI_AVX512VBMI_VL_PermuteVar32x8, - simdBaseJitType, simdSize); - } - else - { - assert(elementSize == 8); - - cnsNode = gtNewIconNode(control); - retNode = gtNewSimdHWIntrinsicNode(type, op1, cnsNode, NI_AVX2_Permute4x64, simdBaseJitType, simdSize); - } - } - else if (simdSize == 64) - { - assert(IsBaselineVector512IsaSupportedDebugOnly()); - if (elementSize == 4) - { - for (uint32_t i = 0; i < elementCount; i++) - { - vecCns.u32[i] = (uint8_t)(vecCns.u8[i * elementSize] / elementSize); - } - - op2 = gtNewVconNode(type); - op2->AsVecCon()->gtSimdVal = vecCns; - - // swap the operands to match the encoding requirements - retNode = gtNewSimdHWIntrinsicNode(type, op2, op1, NI_AVX512F_PermuteVar16x32, simdBaseJitType, simdSize); - } - else if (elementSize == 2) - { - for (uint32_t i = 0; i < elementCount; i++) - { - vecCns.u16[i] = (uint8_t)(vecCns.u8[i * elementSize] / elementSize); - } - - op2 = gtNewVconNode(type); - op2->AsVecCon()->gtSimdVal = vecCns; - - // swap the operands to match the encoding requirements - retNode = gtNewSimdHWIntrinsicNode(type, op2, op1, NI_AVX512BW_PermuteVar32x16, simdBaseJitType, simdSize); - } - else if (elementSize == 1) - { - assert(compIsaSupportedDebugOnly(InstructionSet_AVX512VBMI)); - op2 = gtNewVconNode(type); - op2->AsVecCon()->gtSimdVal = vecCns; - - // swap the operands to match the encoding requirements - retNode = gtNewSimdHWIntrinsicNode(type, op2, op1, NI_AVX512VBMI_PermuteVar64x8, simdBaseJitType, simdSize); - } - else - { - assert(elementSize == 8); - - for (uint32_t i = 0; i < elementCount; i++) - { - vecCns.u64[i] = (uint8_t)(vecCns.u8[i * elementSize] / elementSize); - } - - op2 = gtNewVconNode(type); - op2->AsVecCon()->gtSimdVal = vecCns; - - // swap the operands to match the encoding requirements - retNode = gtNewSimdHWIntrinsicNode(type, op2, op1, NI_AVX512F_PermuteVar8x64, simdBaseJitType, simdSize); - } - assert(retNode != nullptr); - - // TODO-XArch-AVX512: Switch to VPERMI2* - if (needsZero) - { - op2 = gtNewVconNode(type); - op2->AsVecCon()->gtSimdVal = mskCns; - retNode = gtNewSimdBinOpNode(GT_AND, type, op2, retNode, simdBaseJitType, simdSize); - } - - return retNode; - } - else - { - if (needsZero && compOpportunisticallyDependsOn(InstructionSet_SSSE3)) - { - simdBaseJitType = varTypeIsUnsigned(simdBaseType) ? CORINFO_TYPE_UBYTE : CORINFO_TYPE_BYTE; - - op2 = gtNewVconNode(type); - op2->AsVecCon()->gtSimd16Val = vecCns.v128[0]; - - return gtNewSimdHWIntrinsicNode(type, op1, op2, NI_SSSE3_Shuffle, simdBaseJitType, simdSize); - } - - if (varTypeIsLong(simdBaseType)) - { - // TYP_LONG and TYP_ULONG don't have their own shuffle/permute instructions and so we'll - // just utilize the path for TYP_DOUBLE for simplicity. We could alternatively break this - // down into a TYP_INT or TYP_UINT based shuffle, but that's additional complexity for no - // real benefit since shuffle gets its own port rather than using the fp specific ports. - - simdBaseJitType = CORINFO_TYPE_DOUBLE; - simdBaseType = TYP_DOUBLE; - } - - cnsNode = gtNewIconNode(control); - - if (varTypeIsIntegral(simdBaseType)) - { - retNode = gtNewSimdHWIntrinsicNode(type, op1, cnsNode, NI_SSE2_Shuffle, simdBaseJitType, simdSize); - } - else if (compOpportunisticallyDependsOn(InstructionSet_AVX)) - { - retNode = gtNewSimdHWIntrinsicNode(type, op1, cnsNode, NI_AVX_Permute, simdBaseJitType, simdSize); - } - else - { - // for double we need SSE2, but we can't use the integral path ^ because we still need op1Dup here - NamedIntrinsic ni = simdBaseType == TYP_DOUBLE ? NI_SSE2_Shuffle : NI_SSE_Shuffle; - GenTree* op1Dup = fgMakeMultiUse(&op1); - retNode = gtNewSimdHWIntrinsicNode(type, op1, op1Dup, cnsNode, ni, simdBaseJitType, simdSize); - } - } - - assert(retNode != nullptr); - - if (needsZero) - { - assert((simdSize == 32) || !compIsaSupportedDebugOnly(InstructionSet_SSSE3)); - - op2 = gtNewVconNode(type); - op2->AsVecCon()->gtSimdVal = mskCns; - retNode = gtNewSimdBinOpNode(GT_AND, type, op2, retNode, simdBaseJitType, simdSize); - } - - return retNode; -#elif defined(TARGET_ARM64) - uint64_t value = 0; - simd_t vecCns = {}; - - if (simdSize == 16) - { - // Vector128.Shuffle(a, Vector128.Create(2, 3, 0, 1)) -> ExtractVector128(v.AsUInt64(), v.AsUInt64(), 1) - if ((op2->GetIntegralVectorConstElement(0, TYP_ULONG) == 0x300000002) && - (op2->GetIntegralVectorConstElement(1, TYP_ULONG) == 0x100000000)) - { - GenTree* op1Clone = fgMakeMultiUse(&op1); - return gtNewSimdHWIntrinsicNode(type, op1, op1Clone, gtNewIconNode(1), NI_AdvSimd_ExtractVector128, - CORINFO_TYPE_ULONG, simdSize); - } - } - - for (size_t index = 0; index < elementCount; index++) - { - value = op2->GetIntegralVectorConstElement(index, simdBaseType); - - if (value < elementCount) - { - for (uint32_t i = 0; i < elementSize; i++) - { - vecCns.u8[(index * elementSize) + i] = (uint8_t)((value * elementSize) + i); - } - } - else - { - for (uint32_t i = 0; i < elementSize; i++) - { - vecCns.u8[(index * elementSize) + i] = 0xFF; - } - } - } - - NamedIntrinsic lookupIntrinsic = NI_AdvSimd_VectorTableLookup; - - if (simdSize == 16) - { - lookupIntrinsic = NI_AdvSimd_Arm64_VectorTableLookup; - } - - // VectorTableLookup is only valid on byte/sbyte - simdBaseJitType = varTypeIsUnsigned(simdBaseType) ? CORINFO_TYPE_UBYTE : CORINFO_TYPE_BYTE; - - op2 = gtNewVconNode(type); - op2->AsVecCon()->gtSimdVal = vecCns; - - return gtNewSimdHWIntrinsicNode(type, op1, op2, lookupIntrinsic, simdBaseJitType, simdSize); -#else -#error Unsupported platform -#endif // !TARGET_XARCH && !TARGET_ARM64 -} - -GenTree* Compiler::gtNewSimdSqrtNode(var_types type, GenTree* op1, CorInfoType simdBaseJitType, unsigned simdSize) -{ - assert(IsBaselineSimdIsaSupportedDebugOnly()); - - assert(varTypeIsSIMD(type)); - assert(getSIMDTypeForSize(simdSize) == type); - - assert(op1 != nullptr); - assert(op1->TypeIs(type)); - - var_types simdBaseType = JitType2PreciseVarType(simdBaseJitType); - assert(varTypeIsFloating(simdBaseType)); - - NamedIntrinsic intrinsic = NI_Illegal; - -#if defined(TARGET_XARCH) - if (simdSize == 32) - { - assert(compIsaSupportedDebugOnly(InstructionSet_AVX)); - intrinsic = NI_AVX_Sqrt; - } - else if (simdSize == 64) - { - assert(compIsaSupportedDebugOnly(InstructionSet_AVX512F)); - intrinsic = NI_AVX512F_Sqrt; - } - else if (simdBaseType == TYP_FLOAT) - { - intrinsic = NI_SSE_Sqrt; - } - else - { - intrinsic = NI_SSE2_Sqrt; - } -#elif defined(TARGET_ARM64) - if ((simdSize == 8) && (simdBaseType == TYP_DOUBLE)) - { - intrinsic = NI_AdvSimd_SqrtScalar; - } - else - { - intrinsic = NI_AdvSimd_Arm64_Sqrt; - } -#else -#error Unsupported platform -#endif // !TARGET_XARCH && !TARGET_ARM64 - - assert(intrinsic != NI_Illegal); - return gtNewSimdHWIntrinsicNode(type, op1, intrinsic, simdBaseJitType, simdSize); -} - -//---------------------------------------------------------------------------------------------- -// Compiler::gtNewSimdStoreNode: Creates a new simd Store node -// -// Arguments: -// op1 - The address to which op2 is stored -// op2 - The SIMD value to be stored at op1 -// simdBaseJitType - The base JIT type of SIMD type of the intrinsic -// simdSize - The size of the SIMD type of the intrinsic -// -// Returns: -// The created Store node -// -GenTree* Compiler::gtNewSimdStoreNode(GenTree* op1, GenTree* op2, CorInfoType simdBaseJitType, unsigned simdSize) -{ - assert(op1 != nullptr); - assert(op2 != nullptr); - - assert(varTypeIsSIMD(op2)); - assert(getSIMDTypeForSize(simdSize) == op2->TypeGet()); - - var_types simdBaseType = JitType2PreciseVarType(simdBaseJitType); - assert(varTypeIsArithmetic(simdBaseType)); - - return gtNewStoreIndNode(op2->TypeGet(), op1, op2); -} - -//---------------------------------------------------------------------------------------------- -// Compiler::gtNewSimdStoreAlignedNode: Creates a new simd StoreAligned node -// -// Arguments: -// op1 - The address to which op2 is stored -// op2 - The SIMD value to be stored at op1 -// simdBaseJitType - The base JIT type of SIMD type of the intrinsic -// simdSize - The size of the SIMD type of the intrinsic -// -// Returns: -// The created StoreAligned node -// -GenTree* Compiler::gtNewSimdStoreAlignedNode(GenTree* op1, GenTree* op2, CorInfoType simdBaseJitType, unsigned simdSize) -{ -#if defined(TARGET_XARCH) - assert(IsBaselineSimdIsaSupportedDebugOnly()); - - assert(op1 != nullptr); - assert(op2 != nullptr); - - assert(varTypeIsSIMD(op2)); - assert(getSIMDTypeForSize(simdSize) == op2->TypeGet()); - - var_types simdBaseType = JitType2PreciseVarType(simdBaseJitType); - assert(varTypeIsArithmetic(simdBaseType)); - - NamedIntrinsic intrinsic = NI_Illegal; - - if (simdSize == 32) - { - assert(compIsaSupportedDebugOnly(InstructionSet_AVX)); - intrinsic = NI_AVX_StoreAligned; - } - else if (simdSize == 64) - { - assert(compIsaSupportedDebugOnly(InstructionSet_AVX512F)); - intrinsic = NI_AVX512F_StoreAligned; - } - else if (simdBaseType != TYP_FLOAT) - { - intrinsic = NI_SSE2_StoreAligned; - } - else - { - intrinsic = NI_SSE_StoreAligned; - } - - return gtNewSimdHWIntrinsicNode(TYP_VOID, op1, op2, intrinsic, simdBaseJitType, simdSize); -#elif defined(TARGET_ARM64) - // ARM64 doesn't have aligned stores, but aligned stores are only validated to be - // aligned when optimizations are disable, so only skip the intrinsic handling - // if optimizations are enabled - - assert(opts.OptimizationEnabled()); - return gtNewSimdStoreNode(op1, op2, simdBaseJitType, simdSize); -#else -#error Unsupported platform -#endif // !TARGET_XARCH && !TARGET_ARM64 -} - -//---------------------------------------------------------------------------------------------- -// Compiler::gtNewSimdStoreNonTemporalNode: Creates a new simd StoreNonTemporal node -// -// Arguments: -// op1 - The address to which op2 is stored -// op2 - The SIMD value to be stored at op1 -// simdBaseJitType - The base JIT type of SIMD type of the intrinsic -// simdSize - The size of the SIMD type of the intrinsic -// -// Returns: -// The created StoreNonTemporal node -// -GenTree* Compiler::gtNewSimdStoreNonTemporalNode(GenTree* op1, - GenTree* op2, - CorInfoType simdBaseJitType, - unsigned simdSize) -{ -#if defined(TARGET_XARCH) - assert(IsBaselineSimdIsaSupportedDebugOnly()); - - assert(op1 != nullptr); - assert(op2 != nullptr); - - assert(varTypeIsSIMD(op2)); - assert(getSIMDTypeForSize(simdSize) == op2->TypeGet()); - - var_types simdBaseType = JitType2PreciseVarType(simdBaseJitType); - assert(varTypeIsArithmetic(simdBaseType)); - - NamedIntrinsic intrinsic = NI_Illegal; - - if (simdSize == 64) - { - assert(compIsaSupportedDebugOnly(InstructionSet_AVX512F)); - intrinsic = NI_AVX512F_StoreAlignedNonTemporal; - } - else if (simdSize == 32) - { - assert(compIsaSupportedDebugOnly(InstructionSet_AVX)); - intrinsic = NI_AVX_StoreAlignedNonTemporal; - } - else if (simdBaseType != TYP_FLOAT) - { - intrinsic = NI_SSE2_StoreAlignedNonTemporal; - } - else - { - intrinsic = NI_SSE_StoreAlignedNonTemporal; - } - - return gtNewSimdHWIntrinsicNode(TYP_VOID, op1, op2, intrinsic, simdBaseJitType, simdSize); -#elif defined(TARGET_ARM64) - // ARM64 doesn't have aligned stores, but aligned stores are only validated to be - // aligned when optimizations are disable, so only skip the intrinsic handling - // if optimizations are enabled - - assert(opts.OptimizationEnabled()); - return gtNewSimdStoreNode(op1, op2, simdBaseJitType, simdSize); -#else -#error Unsupported platform -#endif // !TARGET_XARCH && !TARGET_ARM64 -} - -GenTree* Compiler::gtNewSimdSumNode(var_types type, GenTree* op1, CorInfoType simdBaseJitType, unsigned simdSize) -{ - assert(IsBaselineSimdIsaSupportedDebugOnly()); - - var_types simdType = getSIMDTypeForSize(simdSize); - assert(varTypeIsSIMD(simdType)); - - assert(op1 != nullptr); - assert(op1->TypeIs(simdType)); - - var_types simdBaseType = JitType2PreciseVarType(simdBaseJitType); - assert(varTypeIsArithmetic(simdBaseType)); - - NamedIntrinsic intrinsic = NI_Illegal; - GenTree* tmp = nullptr; - -#if defined(TARGET_XARCH) - - if (simdSize == 64) - { - assert(IsBaselineVector512IsaSupportedDebugOnly()); - GenTree* op1Dup = fgMakeMultiUse(&op1); - - op1 = gtNewSimdGetLowerNode(TYP_SIMD32, op1, simdBaseJitType, simdSize); - op1Dup = gtNewSimdGetUpperNode(TYP_SIMD32, op1Dup, simdBaseJitType, simdSize); - - if (varTypeIsFloating(simdBaseType)) - { - // We need to ensure deterministic results which requires - // consistently adding values together. Since many operations - // end up operating on 128-bit lanes, we break sum the same way. - - op1 = gtNewSimdSumNode(type, op1, simdBaseJitType, 32); - op1Dup = gtNewSimdSumNode(type, op1Dup, simdBaseJitType, 32); - - return gtNewOperNode(GT_ADD, type, op1, op1Dup); - } - - simdSize = 32; - op1 = gtNewSimdBinOpNode(GT_ADD, TYP_SIMD32, op1, op1Dup, simdBaseJitType, 32); - } - - if (simdSize == 32) - { - assert(compIsaSupportedDebugOnly(InstructionSet_AVX2)); - GenTree* op1Dup = fgMakeMultiUse(&op1); - - op1 = gtNewSimdGetLowerNode(TYP_SIMD16, op1, simdBaseJitType, simdSize); - op1Dup = gtNewSimdGetUpperNode(TYP_SIMD16, op1Dup, simdBaseJitType, simdSize); - - if (varTypeIsFloating(simdBaseType)) - { - // We need to ensure deterministic results which requires - // consistently adding values together. Since many operations - // end up operating on 128-bit lanes, we break sum the same way. - - op1 = gtNewSimdSumNode(type, op1, simdBaseJitType, 16); - op1Dup = gtNewSimdSumNode(type, op1Dup, simdBaseJitType, 16); - - return gtNewOperNode(GT_ADD, type, op1, op1Dup); - } - - simdSize = 16; - op1 = gtNewSimdBinOpNode(GT_ADD, TYP_SIMD16, op1, op1Dup, simdBaseJitType, 16); - } - - assert(simdSize == 16); - - if (varTypeIsFloating(simdBaseType)) - { - if (simdBaseType == TYP_FLOAT) - { - assert(compIsaSupportedDebugOnly(InstructionSet_SSE2)); - GenTree* op1Shuffled = fgMakeMultiUse(&op1); - - if (compOpportunisticallyDependsOn(InstructionSet_AVX)) - { - assert(compIsaSupportedDebugOnly(InstructionSet_AVX)); - // The permute below gives us [0, 1, 2, 3] -> [1, 0, 3, 2] - op1 = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, gtNewIconNode((int)0b10110001, TYP_INT), NI_AVX_Permute, - simdBaseJitType, simdSize); - // The add below now results in [0 + 1, 1 + 0, 2 + 3, 3 + 2] - op1 = gtNewSimdBinOpNode(GT_ADD, TYP_SIMD16, op1, op1Shuffled, simdBaseJitType, simdSize); - op1Shuffled = fgMakeMultiUse(&op1); - // The permute below gives us [0 + 1, 1 + 0, 2 + 3, 3 + 2] -> [2 + 3, 3 + 2, 0 + 1, 1 + 0] - op1 = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, gtNewIconNode((int)0b01001110, TYP_INT), NI_AVX_Permute, - simdBaseJitType, simdSize); - } - else - { - assert(compIsaSupportedDebugOnly(InstructionSet_SSE)); - // The shuffle below gives us [0, 1, 2, 3] -> [1, 0, 3, 2] - op1 = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, op1Shuffled, gtNewIconNode((int)0b10110001, TYP_INT), - NI_SSE_Shuffle, simdBaseJitType, simdSize); - op1Shuffled = fgMakeMultiUse(&op1Shuffled); - // The add below now results in [0 + 1, 1 + 0, 2 + 3, 3 + 2] - op1 = gtNewSimdBinOpNode(GT_ADD, TYP_SIMD16, op1, op1Shuffled, simdBaseJitType, simdSize); - op1Shuffled = fgMakeMultiUse(&op1); - // The shuffle below gives us [0 + 1, 1 + 0, 2 + 3, 3 + 2] -> [2 + 3, 3 + 2, 0 + 1, 1 + 0] - op1 = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, op1Shuffled, gtNewIconNode((int)0b01001110, TYP_INT), - NI_SSE_Shuffle, simdBaseJitType, simdSize); - op1Shuffled = fgMakeMultiUse(&op1Shuffled); - } - // Finally adding the results gets us [(0 + 1) + (2 + 3), (1 + 0) + (3 + 2), (2 + 3) + (0 + 1), (3 + 2) + (1 - // + 0)] - op1 = gtNewSimdBinOpNode(GT_ADD, TYP_SIMD16, op1, op1Shuffled, simdBaseJitType, simdSize); - return gtNewSimdToScalarNode(type, op1, simdBaseJitType, simdSize); - } - else - { - assert(compIsaSupportedDebugOnly(InstructionSet_SSE2)); - GenTree* op1Shuffled = fgMakeMultiUse(&op1); - - if (compOpportunisticallyDependsOn(InstructionSet_AVX)) - { - assert(compIsaSupportedDebugOnly(InstructionSet_AVX)); - // The permute below gives us [0, 1] -> [1, 0] - op1 = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, gtNewIconNode((int)0b0001, TYP_INT), NI_AVX_Permute, - simdBaseJitType, simdSize); - } - else - { - // The shuffle below gives us [0, 1] -> [1, 0] - op1 = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, op1Shuffled, gtNewIconNode((int)0b0001, TYP_INT), - NI_SSE2_Shuffle, simdBaseJitType, simdSize); - op1Shuffled = fgMakeMultiUse(&op1Shuffled); - } - // Finally adding the results gets us [0 + 1, 1 + 0] - op1 = gtNewSimdBinOpNode(GT_ADD, TYP_SIMD16, op1, op1Shuffled, simdBaseJitType, simdSize); - return gtNewSimdToScalarNode(type, op1, simdBaseJitType, simdSize); - } - } - - unsigned vectorLength = getSIMDVectorLength(simdSize, simdBaseType); - int shiftCount = genLog2(vectorLength); - int typeSize = genTypeSize(simdBaseType); - int shiftVal = (typeSize * vectorLength) / 2; - - // The reduced sum is calculated for integer values using a combination of shift + add - // For e.g. consider a 32 bit integer. This means we have 4 values in a XMM register - // After the first shift + add -> [(0 + 2), (1 + 3), ...] - // After the second shift + add -> [(0 + 2 + 1 + 3), ...] - GenTree* opShifted = nullptr; - while (shiftVal >= typeSize) - { - tmp = fgMakeMultiUse(&op1); - opShifted = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, gtNewIconNode(shiftVal, TYP_INT), - NI_SSE2_ShiftRightLogical128BitLane, simdBaseJitType, simdSize); - op1 = gtNewSimdBinOpNode(GT_ADD, TYP_SIMD16, opShifted, tmp, simdBaseJitType, simdSize); - shiftVal = shiftVal / 2; - } - - return gtNewSimdToScalarNode(type, op1, simdBaseJitType, simdSize); - -#elif defined(TARGET_ARM64) - switch (simdBaseType) - { - case TYP_BYTE: - case TYP_UBYTE: - case TYP_SHORT: - case TYP_USHORT: - { - tmp = gtNewSimdHWIntrinsicNode(TYP_SIMD8, op1, NI_AdvSimd_Arm64_AddAcross, simdBaseJitType, simdSize); - return gtNewSimdToScalarNode(type, tmp, simdBaseJitType, 8); - } - - case TYP_INT: - case TYP_UINT: - { - if (simdSize == 8) - { - tmp = fgMakeMultiUse(&op1); - tmp = gtNewSimdHWIntrinsicNode(simdType, op1, tmp, NI_AdvSimd_AddPairwise, simdBaseJitType, simdSize); - } - else - { - tmp = gtNewSimdHWIntrinsicNode(TYP_SIMD8, op1, NI_AdvSimd_Arm64_AddAcross, simdBaseJitType, 16); - } - return gtNewSimdToScalarNode(type, tmp, simdBaseJitType, 8); - } - - case TYP_FLOAT: - { - if (simdSize == 8) - { - op1 = gtNewSimdHWIntrinsicNode(TYP_SIMD8, op1, NI_AdvSimd_Arm64_AddPairwiseScalar, simdBaseJitType, - simdSize); - } - else - { - unsigned vectorLength = getSIMDVectorLength(simdSize, simdBaseType); - int haddCount = genLog2(vectorLength); - - for (int i = 0; i < haddCount; i++) - { - tmp = fgMakeMultiUse(&op1); - op1 = gtNewSimdHWIntrinsicNode(simdType, op1, tmp, NI_AdvSimd_Arm64_AddPairwise, simdBaseJitType, - simdSize); - } - } - return gtNewSimdToScalarNode(type, op1, simdBaseJitType, simdSize); - } - - case TYP_DOUBLE: - case TYP_LONG: - case TYP_ULONG: - { - if (simdSize == 16) - { - op1 = gtNewSimdHWIntrinsicNode(TYP_SIMD8, op1, NI_AdvSimd_Arm64_AddPairwiseScalar, simdBaseJitType, - simdSize); - } - return gtNewSimdToScalarNode(type, op1, simdBaseJitType, 8); - } - default: - { - unreached(); - } - } -#else -#error Unsupported platform -#endif // !TARGET_XARCH && !TARGET_ARM64 -} - -#if defined(TARGET_XARCH) -//---------------------------------------------------------------------------------------------- -// Compiler::gtNewSimdTernaryLogicNode: Creates a new simd TernaryLogic node -// -// Arguments: -// type - The return type of SIMD node being created -// op1 - The value of the first operand: 'A' -// op2 - The value of the second operand: 'B' -// op3 - The value of the third operand: 'C' -// op4 - The constant control byte used to determine how 'A', 'B', and 'C' are manipulated -// simdBaseJitType - The base JIT type of SIMD type of the intrinsic -// simdSize - The size of the SIMD type of the intrinsic -// -// Returns: -// The created TernaryLogic node that performs the specified operation on 'A', 'B', and 'C'. -// -GenTree* Compiler::gtNewSimdTernaryLogicNode(var_types type, - GenTree* op1, - GenTree* op2, - GenTree* op3, - GenTree* op4, - CorInfoType simdBaseJitType, - unsigned simdSize) -{ - assert(IsBaselineVector512IsaSupportedDebugOnly() || - ((simdSize != 64) && compIsaSupportedDebugOnly(InstructionSet_AVX10v1))); - - assert(varTypeIsSIMD(type)); - assert(getSIMDTypeForSize(simdSize) == type); - - assert(op1 != nullptr); - assert(op1->TypeIs(type)); - - assert(op2 != nullptr); - assert(op2->TypeIs(type)); - - assert(op3 != nullptr); - assert(op3->TypeIs(type)); - - assert(op4 != nullptr); - assert(genActualType(op4) == TYP_INT); - - var_types simdBaseType = JitType2PreciseVarType(simdBaseJitType); - assert(varTypeIsArithmetic(simdBaseType)); - - NamedIntrinsic intrinsic = NI_Illegal; - - if (simdSize == 64) - { - intrinsic = NI_AVX512F_TernaryLogic; - } - else if (compOpportunisticallyDependsOn(InstructionSet_AVX10v1)) - { - assert((simdSize == 16) || (simdSize == 32)); - intrinsic = NI_AVX10v1_TernaryLogic; - } - else - { - assert((simdSize == 16) || (simdSize == 32)); - intrinsic = NI_AVX512F_VL_TernaryLogic; - } - -#ifdef TARGET_XARCH - assert(HWIntrinsicInfo::NeedsNormalizeSmallTypeToInt(intrinsic)); - if (varTypeIsSmall(simdBaseType)) - { - simdBaseJitType = varTypeIsUnsigned(simdBaseType) ? CORINFO_TYPE_UINT : CORINFO_TYPE_INT; - simdBaseType = JitType2PreciseVarType(simdBaseJitType); - } -#endif // TARGET_XARCH - - return gtNewSimdHWIntrinsicNode(type, op1, op2, op3, op4, intrinsic, simdBaseJitType, simdSize); -} -#endif // TARGET_XARCH - -//---------------------------------------------------------------------------------------------- -// Compiler::gtNewSimdToScalarNode: Creates a new simd ToScalar node. -// -// Arguments: -// type - The return type of SIMD node being created. -// op1 - The SIMD operand. -// simdBaseJitType - The base JIT type of SIMD type of the intrinsic. -// simdSize - The size of the SIMD type of the intrinsic. -// -// Returns: -// The created node that has the ToScalar implementation. -// -GenTree* Compiler::gtNewSimdToScalarNode(var_types type, GenTree* op1, CorInfoType simdBaseJitType, unsigned simdSize) -{ - assert(IsBaselineSimdIsaSupportedDebugOnly()); - assert(varTypeIsArithmetic(type)); - - assert(op1 != nullptr); - assert(varTypeIsSIMD(op1)); - - var_types simdBaseType = JitType2PreciseVarType(simdBaseJitType); - assert(varTypeIsArithmetic(simdBaseType)); - - NamedIntrinsic intrinsic = NI_Illegal; - -#ifdef TARGET_XARCH -#if defined(TARGET_X86) - if (varTypeIsLong(simdBaseType)) - { - // We need SSE41 to handle long, use software fallback - assert(compIsaSupportedDebugOnly(InstructionSet_SSE41)); - - // Create a GetElement node which handles decomposition - GenTree* op2 = gtNewIconNode(0); - return gtNewSimdGetElementNode(type, op1, op2, simdBaseJitType, simdSize); - } -#endif // TARGET_X86 - - if (simdSize == 64) - { - assert(IsBaselineVector512IsaSupportedDebugOnly()); - intrinsic = NI_Vector512_ToScalar; - } - else if (simdSize == 32) - { - assert(compIsaSupportedDebugOnly(InstructionSet_AVX)); - intrinsic = NI_Vector256_ToScalar; - } - else - { - intrinsic = NI_Vector128_ToScalar; - } -#elif defined(TARGET_ARM64) - if (simdSize == 8) - { - intrinsic = NI_Vector64_ToScalar; - } - else - { - intrinsic = NI_Vector128_ToScalar; - } -#else -#error Unsupported platform -#endif // !TARGET_XARCH && !TARGET_ARM64 - - assert(intrinsic != NI_Illegal); - return gtNewSimdHWIntrinsicNode(type, op1, intrinsic, simdBaseJitType, simdSize); -} - -//------------------------------------------------------------------------ -// gtNewSimdTruncNode: Creates a new simd Truncate node -// -// Arguments: -// type -- The type of the node -// op1 -- The node to truncate -// simdBaseJitType -- the base jit type of the node -// simdSize -- the simd size of the node -// -// Return Value: -// The truncate node -// -GenTree* Compiler::gtNewSimdTruncNode(var_types type, GenTree* op1, CorInfoType simdBaseJitType, unsigned simdSize) -{ - assert(IsBaselineSimdIsaSupportedDebugOnly()); - - assert(varTypeIsSIMD(type)); - assert(getSIMDTypeForSize(simdSize) == type); - - assert(op1 != nullptr); - assert(op1->TypeIs(type)); - - var_types simdBaseType = JitType2PreciseVarType(simdBaseJitType); - assert(varTypeIsFloating(simdBaseType)); - - NamedIntrinsic intrinsic = NI_Illegal; - -#if defined(TARGET_XARCH) - if (simdSize == 32) - { - assert(compIsaSupportedDebugOnly(InstructionSet_AVX)); - intrinsic = NI_AVX_RoundToZero; - } - else if (simdSize == 64) - { - assert(compIsaSupportedDebugOnly(InstructionSet_AVX512F)); - GenTree* op2 = gtNewIconNode(static_cast(FloatRoundingMode::ToZero)); - return gtNewSimdHWIntrinsicNode(type, op1, op2, NI_AVX512F_RoundScale, simdBaseJitType, simdSize); - } - else - { - assert(compIsaSupportedDebugOnly(InstructionSet_SSE41)); - intrinsic = NI_SSE41_RoundToZero; - } -#elif defined(TARGET_ARM64) - if (simdBaseType == TYP_DOUBLE) - { - intrinsic = (simdSize == 8) ? NI_AdvSimd_RoundToZeroScalar : NI_AdvSimd_Arm64_RoundToZero; - } - else - { - intrinsic = NI_AdvSimd_RoundToZero; - } -#else -#error Unsupported platform -#endif // !TARGET_XARCH && !TARGET_ARM64 - - assert(intrinsic != NI_Illegal); - return gtNewSimdHWIntrinsicNode(type, op1, intrinsic, simdBaseJitType, simdSize); -} - -GenTree* Compiler::gtNewSimdUnOpNode( - genTreeOps op, var_types type, GenTree* op1, CorInfoType simdBaseJitType, unsigned simdSize) -{ - assert(IsBaselineSimdIsaSupportedDebugOnly()); - - assert(varTypeIsSIMD(type)); - assert(getSIMDTypeForSize(simdSize) == type); - - assert(op1 != nullptr); - assert(op1->TypeIs(type)); - - var_types simdBaseType = JitType2PreciseVarType(simdBaseJitType); - assert(varTypeIsArithmetic(simdBaseType)); - -#if defined(TARGET_ARM64) - if (op == GT_NEG) - { - switch (simdBaseType) - { - case TYP_UBYTE: - { - simdBaseJitType = CORINFO_TYPE_BYTE; - simdBaseType = TYP_BYTE; - break; - } - - case TYP_USHORT: - { - simdBaseJitType = CORINFO_TYPE_SHORT; - simdBaseType = TYP_SHORT; - break; - } - - case TYP_UINT: - { - simdBaseJitType = CORINFO_TYPE_INT; - simdBaseType = TYP_INT; - break; - } - - case TYP_ULONG: - { - simdBaseJitType = CORINFO_TYPE_LONG; - simdBaseType = TYP_LONG; - break; - } - - default: - { - break; - } - } - } -#endif // TARGET_ARM64 - - NamedIntrinsic intrinsic = - GenTreeHWIntrinsic::GetHWIntrinsicIdForUnOp(this, op, op1, simdBaseType, simdSize, false); - - if (intrinsic != NI_Illegal) - { - return gtNewSimdHWIntrinsicNode(type, op1, intrinsic, simdBaseJitType, simdSize); - } - - switch (op) - { -#if defined(TARGET_XARCH) - case GT_NEG: - { - if (varTypeIsFloating(simdBaseType)) - { - // op1 ^ -0.0 - GenTree* negZero = gtNewDconNode(-0.0, simdBaseType); - negZero = gtNewSimdCreateBroadcastNode(type, negZero, simdBaseJitType, simdSize); - return gtNewSimdBinOpNode(GT_XOR, type, op1, negZero, simdBaseJitType, simdSize); - } - else - { - // Zero - op1 - GenTree* zero = gtNewZeroConNode(type); - return gtNewSimdBinOpNode(GT_SUB, type, zero, op1, simdBaseJitType, simdSize); - } - } - - case GT_NOT: - { - // op1 ^ AllBitsSet - GenTree* allBitsSet = gtNewAllBitsSetConNode(type); - return gtNewSimdBinOpNode(GT_XOR, type, op1, allBitsSet, simdBaseJitType, simdSize); - } -#endif // TARGET_XARCH - - default: - { - unreached(); - } - } -} - -GenTree* Compiler::gtNewSimdWidenLowerNode(var_types type, GenTree* op1, CorInfoType simdBaseJitType, unsigned simdSize) -{ - assert(IsBaselineSimdIsaSupportedDebugOnly()); - - assert(varTypeIsSIMD(type)); - assert(getSIMDTypeForSize(simdSize) == type); - - assert(op1 != nullptr); - assert(op1->TypeIs(type)); - - var_types simdBaseType = JitType2PreciseVarType(simdBaseJitType); - assert(varTypeIsArithmetic(simdBaseType) && !varTypeIsLong(simdBaseType)); - - NamedIntrinsic intrinsic = NI_Illegal; - - GenTree* tmp1; - -#if defined(TARGET_XARCH) - if (simdSize == 64) - { - assert(IsBaselineVector512IsaSupportedDebugOnly()); - - tmp1 = gtNewSimdGetLowerNode(TYP_SIMD32, op1, simdBaseJitType, simdSize); - - switch (simdBaseType) - { - case TYP_BYTE: - { - intrinsic = NI_AVX512BW_ConvertToVector512Int16; - break; - } - - case TYP_UBYTE: - { - intrinsic = NI_AVX512BW_ConvertToVector512UInt16; - break; - } - - case TYP_SHORT: - { - intrinsic = NI_AVX512F_ConvertToVector512Int32; - break; - } - - case TYP_USHORT: - { - intrinsic = NI_AVX512F_ConvertToVector512UInt32; - break; - } - - case TYP_INT: - { - intrinsic = NI_AVX512F_ConvertToVector512Int64; - break; - } - - case TYP_UINT: - { - intrinsic = NI_AVX512F_ConvertToVector512UInt64; - break; - } - - case TYP_FLOAT: - { - intrinsic = NI_AVX512F_ConvertToVector512Double; - break; - } - - default: - { - unreached(); - } - } - - assert(intrinsic != NI_Illegal); - return gtNewSimdHWIntrinsicNode(type, tmp1, intrinsic, simdBaseJitType, simdSize); - } - else if (simdSize == 32) - { - assert(compIsaSupportedDebugOnly(InstructionSet_AVX)); - assert(!varTypeIsIntegral(simdBaseType) || compIsaSupportedDebugOnly(InstructionSet_AVX2)); - - tmp1 = gtNewSimdGetLowerNode(TYP_SIMD16, op1, simdBaseJitType, simdSize); - - switch (simdBaseType) - { - case TYP_BYTE: - case TYP_UBYTE: - { - intrinsic = NI_AVX2_ConvertToVector256Int16; - break; - } - - case TYP_SHORT: - case TYP_USHORT: - { - intrinsic = NI_AVX2_ConvertToVector256Int32; - break; - } - - case TYP_INT: - case TYP_UINT: - { - intrinsic = NI_AVX2_ConvertToVector256Int64; - break; - } - - case TYP_FLOAT: - { - intrinsic = NI_AVX_ConvertToVector256Double; - break; - } - - default: - { - unreached(); - } - } - - assert(intrinsic != NI_Illegal); - return gtNewSimdHWIntrinsicNode(type, tmp1, intrinsic, simdBaseJitType, simdSize); - } - else if ((simdBaseType == TYP_FLOAT) || compOpportunisticallyDependsOn(InstructionSet_SSE41)) - { - switch (simdBaseType) - { - case TYP_BYTE: - case TYP_UBYTE: - { - intrinsic = NI_SSE41_ConvertToVector128Int16; - break; - } - - case TYP_SHORT: - case TYP_USHORT: - { - intrinsic = NI_SSE41_ConvertToVector128Int32; - break; - } - - case TYP_INT: - case TYP_UINT: - { - intrinsic = NI_SSE41_ConvertToVector128Int64; - break; - } - - case TYP_FLOAT: - { - intrinsic = NI_SSE2_ConvertToVector128Double; - break; - } - - default: - { - unreached(); - } - } - - assert(intrinsic != NI_Illegal); - return gtNewSimdHWIntrinsicNode(type, op1, intrinsic, simdBaseJitType, simdSize); - } - else - { - tmp1 = gtNewZeroConNode(type); - - if (varTypeIsSigned(simdBaseType)) - { - GenTree* op1Dup = fgMakeMultiUse(&op1); - - tmp1 = gtNewSimdHWIntrinsicNode(type, op1Dup, tmp1, NI_SSE2_CompareLessThan, simdBaseJitType, simdSize); - } - - return gtNewSimdHWIntrinsicNode(type, op1, tmp1, NI_SSE2_UnpackLow, simdBaseJitType, simdSize); - } -#elif defined(TARGET_ARM64) - if (simdSize == 16) - { - tmp1 = gtNewSimdGetLowerNode(TYP_SIMD8, op1, simdBaseJitType, simdSize); - } - else - { - assert(simdSize == 8); - tmp1 = op1; - } - - if (varTypeIsFloating(simdBaseType)) - { - assert(simdBaseType == TYP_FLOAT); - intrinsic = NI_AdvSimd_Arm64_ConvertToDouble; - } - else if (varTypeIsSigned(simdBaseType)) - { - intrinsic = NI_AdvSimd_SignExtendWideningLower; - } - else - { - intrinsic = NI_AdvSimd_ZeroExtendWideningLower; - } - - assert(intrinsic != NI_Illegal); - tmp1 = gtNewSimdHWIntrinsicNode(TYP_SIMD16, tmp1, intrinsic, simdBaseJitType, 8); - - if (simdSize == 8) - { - tmp1 = gtNewSimdGetLowerNode(TYP_SIMD8, tmp1, simdBaseJitType, 16); - } - - return tmp1; -#else -#error Unsupported platform -#endif // !TARGET_XARCH && !TARGET_ARM64 -} - -GenTree* Compiler::gtNewSimdWidenUpperNode(var_types type, GenTree* op1, CorInfoType simdBaseJitType, unsigned simdSize) -{ - assert(IsBaselineSimdIsaSupportedDebugOnly()); - - assert(varTypeIsSIMD(type)); - assert(getSIMDTypeForSize(simdSize) == type); - - assert(op1 != nullptr); - assert(op1->TypeIs(type)); - - var_types simdBaseType = JitType2PreciseVarType(simdBaseJitType); - assert(varTypeIsArithmetic(simdBaseType) && !varTypeIsLong(simdBaseType)); - - NamedIntrinsic intrinsic = NI_Illegal; - - GenTree* tmp1; - -#if defined(TARGET_XARCH) - if (simdSize == 64) - { - assert(IsBaselineVector512IsaSupportedDebugOnly()); - - tmp1 = gtNewSimdGetUpperNode(TYP_SIMD32, op1, simdBaseJitType, simdSize); - - switch (simdBaseType) - { - case TYP_BYTE: - { - intrinsic = NI_AVX512BW_ConvertToVector512Int16; - break; - } - - case TYP_UBYTE: - { - intrinsic = NI_AVX512BW_ConvertToVector512UInt16; - break; - } - - case TYP_SHORT: - { - intrinsic = NI_AVX512F_ConvertToVector512Int32; - break; - } - - case TYP_USHORT: - { - intrinsic = NI_AVX512F_ConvertToVector512UInt32; - break; - } - - case TYP_INT: - { - intrinsic = NI_AVX512F_ConvertToVector512Int64; - break; - } - - case TYP_UINT: - { - intrinsic = NI_AVX512F_ConvertToVector512UInt64; - break; - } - - case TYP_FLOAT: - { - intrinsic = NI_AVX512F_ConvertToVector512Double; - break; - } - - default: - { - unreached(); - } - } - - assert(intrinsic != NI_Illegal); - return gtNewSimdHWIntrinsicNode(type, tmp1, intrinsic, simdBaseJitType, simdSize); - } - else if (simdSize == 32) - { - assert(compIsaSupportedDebugOnly(InstructionSet_AVX)); - assert(!varTypeIsIntegral(simdBaseType) || compIsaSupportedDebugOnly(InstructionSet_AVX2)); - - tmp1 = gtNewSimdGetUpperNode(TYP_SIMD16, op1, simdBaseJitType, simdSize); - - switch (simdBaseType) - { - case TYP_BYTE: - case TYP_UBYTE: - { - intrinsic = NI_AVX2_ConvertToVector256Int16; - break; - } - - case TYP_SHORT: - case TYP_USHORT: - { - intrinsic = NI_AVX2_ConvertToVector256Int32; - break; - } - - case TYP_INT: - case TYP_UINT: - { - intrinsic = NI_AVX2_ConvertToVector256Int64; - break; - } - - case TYP_FLOAT: - { - intrinsic = NI_AVX_ConvertToVector256Double; - break; - } - - default: - { - unreached(); - } - } - - assert(intrinsic != NI_Illegal); - return gtNewSimdHWIntrinsicNode(type, tmp1, intrinsic, simdBaseJitType, simdSize); - } - else if (varTypeIsFloating(simdBaseType)) - { - assert(simdBaseType == TYP_FLOAT); - - GenTree* op1Dup = fgMakeMultiUse(&op1); - - tmp1 = gtNewSimdHWIntrinsicNode(type, op1, op1Dup, NI_SSE_MoveHighToLow, simdBaseJitType, simdSize); - return gtNewSimdHWIntrinsicNode(type, tmp1, NI_SSE2_ConvertToVector128Double, simdBaseJitType, simdSize); - } - else if (compOpportunisticallyDependsOn(InstructionSet_SSE41)) - { - tmp1 = gtNewSimdHWIntrinsicNode(type, op1, gtNewIconNode(8), NI_SSE2_ShiftRightLogical128BitLane, - simdBaseJitType, simdSize); - - switch (simdBaseType) - { - case TYP_BYTE: - case TYP_UBYTE: - { - intrinsic = NI_SSE41_ConvertToVector128Int16; - break; - } - - case TYP_SHORT: - case TYP_USHORT: - { - intrinsic = NI_SSE41_ConvertToVector128Int32; - break; - } - - case TYP_INT: - case TYP_UINT: - { - intrinsic = NI_SSE41_ConvertToVector128Int64; - break; - } - - default: - { - unreached(); - } - } - - assert(intrinsic != NI_Illegal); - return gtNewSimdHWIntrinsicNode(type, tmp1, intrinsic, simdBaseJitType, simdSize); - } - else - { - tmp1 = gtNewZeroConNode(type); - - if (varTypeIsSigned(simdBaseType)) - { - GenTree* op1Dup = fgMakeMultiUse(&op1); - - tmp1 = gtNewSimdHWIntrinsicNode(type, op1Dup, tmp1, NI_SSE2_CompareLessThan, simdBaseJitType, simdSize); - } - - return gtNewSimdHWIntrinsicNode(type, op1, tmp1, NI_SSE2_UnpackHigh, simdBaseJitType, simdSize); - } -#elif defined(TARGET_ARM64) - if (simdSize == 16) - { - if (varTypeIsFloating(simdBaseType)) - { - assert(simdBaseType == TYP_FLOAT); - intrinsic = NI_AdvSimd_Arm64_ConvertToDoubleUpper; - } - else if (varTypeIsSigned(simdBaseType)) - { - intrinsic = NI_AdvSimd_SignExtendWideningUpper; - } - else - { - intrinsic = NI_AdvSimd_ZeroExtendWideningUpper; - } - - assert(intrinsic != NI_Illegal); - return gtNewSimdHWIntrinsicNode(type, op1, intrinsic, simdBaseJitType, simdSize); - } - else - { - assert(simdSize == 8); - ssize_t index = 8 / genTypeSize(simdBaseType); - - if (varTypeIsFloating(simdBaseType)) - { - assert(simdBaseType == TYP_FLOAT); - intrinsic = NI_AdvSimd_Arm64_ConvertToDouble; - } - else if (varTypeIsSigned(simdBaseType)) - { - intrinsic = NI_AdvSimd_SignExtendWideningLower; - } - else - { - intrinsic = NI_AdvSimd_ZeroExtendWideningLower; - } - - assert(intrinsic != NI_Illegal); - - tmp1 = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, intrinsic, simdBaseJitType, simdSize); - return gtNewSimdGetUpperNode(TYP_SIMD8, tmp1, simdBaseJitType, 16); - } -#else -#error Unsupported platform -#endif // !TARGET_XARCH && !TARGET_ARM64 -} - -GenTree* Compiler::gtNewSimdWithElementNode( - var_types type, GenTree* op1, GenTree* op2, GenTree* op3, CorInfoType simdBaseJitType, unsigned simdSize) -{ - NamedIntrinsic hwIntrinsicID = NI_Vector128_WithElement; - var_types simdBaseType = JitType2PreciseVarType(simdBaseJitType); - - assert(varTypeIsArithmetic(simdBaseType)); - assert(op2->IsCnsIntOrI()); - assert(varTypeIsArithmetic(op3)); - - ssize_t imm8 = op2->AsIntCon()->IconValue(); - ssize_t count = simdSize / genTypeSize(simdBaseType); - - assert((0 <= imm8) && (imm8 < count)); - -#if defined(TARGET_XARCH) - switch (simdBaseType) - { - // Using software fallback if simdBaseType is not supported by hardware - case TYP_BYTE: - case TYP_UBYTE: - case TYP_INT: - case TYP_UINT: - assert(compIsaSupportedDebugOnly(InstructionSet_SSE41)); - break; - - case TYP_LONG: - case TYP_ULONG: - assert(compIsaSupportedDebugOnly(InstructionSet_SSE41_X64)); - break; - - case TYP_DOUBLE: - case TYP_FLOAT: - case TYP_SHORT: - case TYP_USHORT: - assert(compIsaSupportedDebugOnly(InstructionSet_SSE2)); - break; - - default: - unreached(); - } - - if (simdSize == 64) - { - hwIntrinsicID = NI_Vector512_WithElement; - } - else if (simdSize == 32) - { - hwIntrinsicID = NI_Vector256_WithElement; - } -#elif defined(TARGET_ARM64) - switch (simdBaseType) - { - case TYP_LONG: - case TYP_ULONG: - case TYP_DOUBLE: - if (simdSize == 8) - { - return gtNewSimdHWIntrinsicNode(type, op3, NI_Vector64_Create, simdBaseJitType, simdSize); - } - break; - - case TYP_FLOAT: - case TYP_BYTE: - case TYP_UBYTE: - case TYP_SHORT: - case TYP_USHORT: - case TYP_INT: - case TYP_UINT: - break; - - default: - unreached(); - } - - hwIntrinsicID = NI_AdvSimd_Insert; -#else -#error Unsupported platform -#endif // !TARGET_XARCH && !TARGET_ARM64 - - return gtNewSimdHWIntrinsicNode(type, op1, op2, op3, hwIntrinsicID, simdBaseJitType, simdSize); -} - -#ifdef TARGET_ARM64 -//------------------------------------------------------------------------ -// gtConvertTableOpToFieldList: Convert a operand that represents table of rows into -// field list, where each field represents a row in the table. -// -// Arguments: -// op -- Operand to convert. -// fieldCount -- Number of fields or rows present. -// -// Return Value: -// The GenTreeFieldList node. -// -GenTreeFieldList* Compiler::gtConvertTableOpToFieldList(GenTree* op, unsigned fieldCount) -{ - LclVarDsc* opVarDsc = lvaGetDesc(op->AsLclVar()); - unsigned lclNum = lvaGetLclNum(opVarDsc); - unsigned fieldSize = opVarDsc->lvSize() / fieldCount; - var_types fieldType = Compiler::getSIMDTypeForSize(fieldSize); - - GenTreeFieldList* fieldList = new (this, GT_FIELD_LIST) GenTreeFieldList(); - int offset = 0; - for (unsigned fieldId = 0; fieldId < fieldCount; fieldId++) - { - GenTreeLclFld* fldNode = gtNewLclFldNode(lclNum, fieldType, offset); - fieldList->AddField(this, fldNode, offset, fieldType); - - offset += fieldSize; - } - return fieldList; -} - -//------------------------------------------------------------------------ -// gtConvertParamOpToFieldList: Convert a operand that represents tuple of struct into -// field list, where each field represents a struct in the tuple. -// -// Arguments: -// op -- Operand to convert. -// fieldCount -- Number of fields or rows present. -// clsHnd -- Class handle of the tuple. -// -// Return Value: -// The GenTreeFieldList node. -// -GenTreeFieldList* Compiler::gtConvertParamOpToFieldList(GenTree* op, unsigned fieldCount, CORINFO_CLASS_HANDLE clsHnd) -{ - LclVarDsc* opVarDsc = lvaGetDesc(op->AsLclVar()); - unsigned lclNum = lvaGetLclNum(opVarDsc); - unsigned fieldSize = opVarDsc->lvSize() / fieldCount; - GenTreeFieldList* fieldList = new (this, GT_FIELD_LIST) GenTreeFieldList(); - int offset = 0; - unsigned sizeBytes = 0; - CORINFO_CLASS_HANDLE structType; - - for (unsigned fieldId = 0; fieldId < fieldCount; fieldId++) - { - CORINFO_FIELD_HANDLE fieldHandle = info.compCompHnd->getFieldInClass(clsHnd, fieldId); - JitType2PreciseVarType(info.compCompHnd->getFieldType(fieldHandle, &structType)); - getBaseJitTypeAndSizeOfSIMDType(structType, &sizeBytes); - var_types simdType = getSIMDTypeForSize(sizeBytes); - - GenTreeLclFld* fldNode = gtNewLclFldNode(lclNum, simdType, offset); - fieldList->AddField(this, fldNode, offset, simdType); - - offset += fieldSize; - } - return fieldList; -} -#endif // TARGET_ARM64 - -GenTree* Compiler::gtNewSimdWithLowerNode( - var_types type, GenTree* op1, GenTree* op2, CorInfoType simdBaseJitType, unsigned simdSize) -{ - var_types simdBaseType = JitType2PreciseVarType(simdBaseJitType); - assert(varTypeIsArithmetic(simdBaseType)); - - NamedIntrinsic intrinsicId = NI_Illegal; - -#if defined(TARGET_XARCH) - if (simdSize == 32) - { - assert(type == TYP_SIMD32); - intrinsicId = NI_Vector256_WithLower; - } - else - { - assert((type == TYP_SIMD64) && (simdSize == 64)); - intrinsicId = NI_Vector512_WithLower; - } -#elif defined(TARGET_ARM64) - assert((type == TYP_SIMD16) && (simdSize == 16)); - intrinsicId = NI_Vector128_WithLower; -#else -#error Unsupported platform -#endif // !TARGET_XARCH && !TARGET_ARM64 - - return gtNewSimdHWIntrinsicNode(type, op1, op2, intrinsicId, simdBaseJitType, simdSize); -} - -GenTree* Compiler::gtNewSimdWithUpperNode( - var_types type, GenTree* op1, GenTree* op2, CorInfoType simdBaseJitType, unsigned simdSize) -{ - var_types simdBaseType = JitType2PreciseVarType(simdBaseJitType); - assert(varTypeIsArithmetic(simdBaseType)); - - NamedIntrinsic intrinsicId = NI_Illegal; - -#if defined(TARGET_XARCH) - if (simdSize == 32) - { - assert(type == TYP_SIMD32); - intrinsicId = NI_Vector256_WithUpper; - } - else - { - assert((type == TYP_SIMD64) && (simdSize == 64)); - intrinsicId = NI_Vector512_WithUpper; - } -#elif defined(TARGET_ARM64) - assert((type == TYP_SIMD16) && (simdSize == 16)); - intrinsicId = NI_Vector128_WithUpper; -#else -#error Unsupported platform -#endif // !TARGET_XARCH && !TARGET_ARM64 - - return gtNewSimdHWIntrinsicNode(type, op1, op2, intrinsicId, simdBaseJitType, simdSize); -} - -GenTreeHWIntrinsic* Compiler::gtNewScalarHWIntrinsicNode(var_types type, NamedIntrinsic hwIntrinsicID) -{ - return new (this, GT_HWINTRINSIC) - GenTreeHWIntrinsic(type, getAllocator(CMK_ASTNode), hwIntrinsicID, CORINFO_TYPE_UNDEF, 0); -} - -GenTreeHWIntrinsic* Compiler::gtNewScalarHWIntrinsicNode(var_types type, GenTree* op1, NamedIntrinsic hwIntrinsicID) -{ - SetOpLclRelatedToSIMDIntrinsic(op1); - - return new (this, GT_HWINTRINSIC) - GenTreeHWIntrinsic(type, getAllocator(CMK_ASTNode), hwIntrinsicID, CORINFO_TYPE_UNDEF, 0, op1); -} - -GenTreeHWIntrinsic* Compiler::gtNewScalarHWIntrinsicNode(var_types type, - GenTree* op1, - GenTree* op2, - NamedIntrinsic hwIntrinsicID) -{ - SetOpLclRelatedToSIMDIntrinsic(op1); - SetOpLclRelatedToSIMDIntrinsic(op2); - - return new (this, GT_HWINTRINSIC) - GenTreeHWIntrinsic(type, getAllocator(CMK_ASTNode), hwIntrinsicID, CORINFO_TYPE_UNDEF, 0, op1, op2); -} - -GenTreeHWIntrinsic* Compiler::gtNewScalarHWIntrinsicNode( - var_types type, GenTree* op1, GenTree* op2, GenTree* op3, NamedIntrinsic hwIntrinsicID) -{ - SetOpLclRelatedToSIMDIntrinsic(op1); - SetOpLclRelatedToSIMDIntrinsic(op2); - SetOpLclRelatedToSIMDIntrinsic(op3); - - return new (this, GT_HWINTRINSIC) - GenTreeHWIntrinsic(type, getAllocator(CMK_ASTNode), hwIntrinsicID, CORINFO_TYPE_UNDEF, 0, op1, op2, op3); -} - -//------------------------------------------------------------------------ -// OperIsHWIntrinsic: Is this a hwintrinsic with the specified id -// -// Arguments: -// intrinsicId -- the id to compare with the current node -// -// Return Value: -// true if the node is a hwintrinsic intrinsic with the specified id -// otherwise; false -// -bool GenTree::OperIsHWIntrinsic(NamedIntrinsic intrinsicId) const -{ - if (OperIsHWIntrinsic()) - { - return AsHWIntrinsic()->OperIsHWIntrinsic(intrinsicId); - } - return false; -} - -//------------------------------------------------------------------------ -// OperIsConvertMaskToVector: Is this a ConvertMaskToVector hwintrinsic -// -// Return Value: -// true if the node is a ConvertMaskToVector hwintrinsic -// otherwise; false -// -bool GenTree::OperIsConvertMaskToVector() const -{ - if (OperIsHWIntrinsic()) - { - return AsHWIntrinsic()->OperIsConvertMaskToVector(); - } - return false; -} - -//------------------------------------------------------------------------ -// OperIsConvertVectorToMask: Is this a ConvertVectorToMask hwintrinsic -// -// Return Value: -// true if the node is a ConvertVectorToMask hwintrinsic -// otherwise; false -// -bool GenTree::OperIsConvertVectorToMask() const -{ - if (OperIsHWIntrinsic()) - { - return AsHWIntrinsic()->OperIsConvertVectorToMask(); - } - return false; -} - -//------------------------------------------------------------------------ -// OperIsVectorConditionalSelect: Is this a vector ConditionalSelect hwintrinsic -// -// Return Value: -// true if the node is a vector ConditionalSelect hwintrinsic -// otherwise; false -// -bool GenTree::OperIsVectorConditionalSelect() const -{ - if (OperIsHWIntrinsic()) - { - return AsHWIntrinsic()->OperIsVectorConditionalSelect(); - } - return false; -} - -//------------------------------------------------------------------------ -// OperIsMemoryLoad: Does this HWI node have memory load semantics? -// -// Arguments: -// pAddr - optional [out] parameter for the address -// -// Return Value: -// Whether this intrinsic may throw NullReferenceException if the -// address is "null". -// -bool GenTreeHWIntrinsic::OperIsMemoryLoad(GenTree** pAddr) const -{ - GenTree* addr = nullptr; - -#if defined(TARGET_XARCH) || defined(TARGET_ARM64) - NamedIntrinsic intrinsicId = GetHWIntrinsicId(); - HWIntrinsicCategory category = HWIntrinsicInfo::lookupCategory(intrinsicId); - - if (category == HW_Category_MemoryLoad) - { - switch (intrinsicId) - { -#ifdef TARGET_XARCH - case NI_SSE_LoadLow: - case NI_SSE_LoadHigh: - case NI_SSE2_LoadLow: - case NI_SSE2_LoadHigh: - addr = Op(2); - break; -#endif // TARGET_XARCH - -#ifdef TARGET_ARM64 - case NI_AdvSimd_LoadAndInsertScalar: - case NI_AdvSimd_LoadAndInsertScalarVector64x2: - case NI_AdvSimd_LoadAndInsertScalarVector64x3: - case NI_AdvSimd_LoadAndInsertScalarVector64x4: - case NI_AdvSimd_Arm64_LoadAndInsertScalarVector128x2: - case NI_AdvSimd_Arm64_LoadAndInsertScalarVector128x3: - case NI_AdvSimd_Arm64_LoadAndInsertScalarVector128x4: - addr = Op(3); - break; - - case NI_Sve_GatherVector: - case NI_Sve_GatherVectorByteZeroExtend: - case NI_Sve_GatherVectorByteZeroExtendFirstFaulting: - case NI_Sve_GatherVectorFirstFaulting: - case NI_Sve_GatherVectorInt16SignExtend: - case NI_Sve_GatherVectorInt16SignExtendFirstFaulting: - case NI_Sve_GatherVectorInt16WithByteOffsetsSignExtend: - case NI_Sve_GatherVectorInt16WithByteOffsetsSignExtendFirstFaulting: - case NI_Sve_GatherVectorInt32SignExtend: - case NI_Sve_GatherVectorInt32SignExtendFirstFaulting: - case NI_Sve_GatherVectorInt32WithByteOffsetsSignExtend: - case NI_Sve_GatherVectorInt32WithByteOffsetsSignExtendFirstFaulting: - case NI_Sve_GatherVectorSByteSignExtend: - case NI_Sve_GatherVectorSByteSignExtendFirstFaulting: - case NI_Sve_GatherVectorUInt16WithByteOffsetsZeroExtend: - case NI_Sve_GatherVectorUInt16WithByteOffsetsZeroExtendFirstFaulting: - case NI_Sve_GatherVectorUInt16ZeroExtend: - case NI_Sve_GatherVectorUInt16ZeroExtendFirstFaulting: - case NI_Sve_GatherVectorUInt32WithByteOffsetsZeroExtend: - case NI_Sve_GatherVectorUInt32WithByteOffsetsZeroExtendFirstFaulting: - case NI_Sve_GatherVectorUInt32ZeroExtend: - case NI_Sve_GatherVectorUInt32ZeroExtendFirstFaulting: - case NI_Sve_GatherVectorWithByteOffsetFirstFaulting: - case NI_Sve_GatherVectorWithByteOffsets: - case NI_Sve_LoadVector: - case NI_Sve_LoadVectorNonTemporal: - case NI_Sve_LoadVector128AndReplicateToVector: - case NI_Sve_LoadVectorByteZeroExtendFirstFaulting: - case NI_Sve_LoadVectorByteZeroExtendToInt16: - case NI_Sve_LoadVectorByteZeroExtendToInt32: - case NI_Sve_LoadVectorByteZeroExtendToInt64: - case NI_Sve_LoadVectorByteZeroExtendToUInt16: - case NI_Sve_LoadVectorByteZeroExtendToUInt32: - case NI_Sve_LoadVectorByteZeroExtendToUInt64: - case NI_Sve_LoadVectorFirstFaulting: - case NI_Sve_LoadVectorInt16SignExtendFirstFaulting: - case NI_Sve_LoadVectorInt16SignExtendToInt32: - case NI_Sve_LoadVectorInt16SignExtendToInt64: - case NI_Sve_LoadVectorInt16SignExtendToUInt32: - case NI_Sve_LoadVectorInt16SignExtendToUInt64: - case NI_Sve_LoadVectorInt32SignExtendFirstFaulting: - case NI_Sve_LoadVectorInt32SignExtendToInt64: - case NI_Sve_LoadVectorInt32SignExtendToUInt64: - case NI_Sve_LoadVectorSByteSignExtendFirstFaulting: - case NI_Sve_LoadVectorSByteSignExtendToInt16: - case NI_Sve_LoadVectorSByteSignExtendToInt32: - case NI_Sve_LoadVectorSByteSignExtendToInt64: - case NI_Sve_LoadVectorSByteSignExtendToUInt16: - case NI_Sve_LoadVectorSByteSignExtendToUInt32: - case NI_Sve_LoadVectorSByteSignExtendToUInt64: - case NI_Sve_LoadVectorUInt16ZeroExtendFirstFaulting: - case NI_Sve_LoadVectorUInt16ZeroExtendToInt32: - case NI_Sve_LoadVectorUInt16ZeroExtendToInt64: - case NI_Sve_LoadVectorUInt16ZeroExtendToUInt32: - case NI_Sve_LoadVectorUInt16ZeroExtendToUInt64: - case NI_Sve_LoadVectorUInt32ZeroExtendFirstFaulting: - case NI_Sve_LoadVectorUInt32ZeroExtendToInt64: - case NI_Sve_LoadVectorUInt32ZeroExtendToUInt64: - case NI_Sve_Load2xVectorAndUnzip: - case NI_Sve_Load3xVectorAndUnzip: - case NI_Sve_Load4xVectorAndUnzip: - addr = Op(2); - break; - -#endif // TARGET_ARM64 - - default: - addr = Op(1); - break; - } - } -#ifdef TARGET_XARCH - else if (HWIntrinsicInfo::MaybeMemoryLoad(intrinsicId)) - { - // Some intrinsics (without HW_Category_MemoryLoad) also have MemoryLoad semantics - // This is generally because they have both vector and pointer overloads, e.g., - // * Vector128 BroadcastScalarToVector128(Vector128 value) - // * Vector128 BroadcastScalarToVector128(byte* source) - // - if ((category == HW_Category_SimpleSIMD) || (category == HW_Category_SIMDScalar)) - { - assert(GetOperandCount() == 1); - - switch (intrinsicId) - { - case NI_SSE41_ConvertToVector128Int16: - case NI_SSE41_ConvertToVector128Int32: - case NI_SSE41_ConvertToVector128Int64: - case NI_AVX2_BroadcastScalarToVector128: - case NI_AVX2_BroadcastScalarToVector256: - case NI_AVX2_ConvertToVector256Int16: - case NI_AVX2_ConvertToVector256Int32: - case NI_AVX2_ConvertToVector256Int64: - case NI_AVX2_BroadcastVector128ToVector256: - case NI_AVX512F_BroadcastVector128ToVector512: - case NI_AVX512F_BroadcastVector256ToVector512: - if (GetAuxiliaryJitType() == CORINFO_TYPE_PTR) - { - addr = Op(1); - } - else - { - assert(GetAuxiliaryJitType() == CORINFO_TYPE_UNDEF); - } - break; - - default: - unreached(); - } - } - else if (category == HW_Category_IMM) - { - switch (intrinsicId) - { - case NI_AVX2_GatherVector128: - case NI_AVX2_GatherVector256: - addr = Op(1); - break; - - case NI_AVX2_GatherMaskVector128: - case NI_AVX2_GatherMaskVector256: - addr = Op(2); - break; - - default: - break; - } - } - } -#endif // TARGET_XARCH -#endif // TARGET_XARCH || TARGET_ARM64 - - if (pAddr != nullptr) - { - *pAddr = addr; - } - - if (addr != nullptr) - { -#ifdef TARGET_ARM64 - static_assert_no_msg( - AreContiguous(NI_Sve_GatherVector, NI_Sve_GatherVectorByteZeroExtend, - NI_Sve_GatherVectorByteZeroExtendFirstFaulting, NI_Sve_GatherVectorFirstFaulting, - NI_Sve_GatherVectorInt16SignExtend, NI_Sve_GatherVectorInt16SignExtendFirstFaulting, - NI_Sve_GatherVectorInt16WithByteOffsetsSignExtend, - NI_Sve_GatherVectorInt16WithByteOffsetsSignExtendFirstFaulting, - NI_Sve_GatherVectorInt32SignExtend, NI_Sve_GatherVectorInt32SignExtendFirstFaulting, - NI_Sve_GatherVectorInt32WithByteOffsetsSignExtend, - NI_Sve_GatherVectorInt32WithByteOffsetsSignExtendFirstFaulting, - NI_Sve_GatherVectorSByteSignExtend, NI_Sve_GatherVectorSByteSignExtendFirstFaulting, - NI_Sve_GatherVectorUInt16WithByteOffsetsZeroExtend, - NI_Sve_GatherVectorUInt16WithByteOffsetsZeroExtendFirstFaulting, - NI_Sve_GatherVectorUInt16ZeroExtend, NI_Sve_GatherVectorUInt16ZeroExtendFirstFaulting, - NI_Sve_GatherVectorUInt32WithByteOffsetsZeroExtend, - NI_Sve_GatherVectorUInt32WithByteOffsetsZeroExtendFirstFaulting, - NI_Sve_GatherVectorUInt32ZeroExtend, NI_Sve_GatherVectorUInt32ZeroExtendFirstFaulting)); - assert(varTypeIsI(addr) || - (varTypeIsSIMD(addr) && ((intrinsicId >= NI_Sve_GatherVector) && - (intrinsicId <= NI_Sve_GatherVectorUInt32ZeroExtendFirstFaulting)))); -#else - assert(varTypeIsI(addr)); -#endif - return true; - } - - return false; -} - -//------------------------------------------------------------------------ -// OperIsMemoryLoad: Does this HWI node have memory store semantics? -// -// Arguments: -// pAddr - optional [out] parameter for the address -// -// Return Value: -// Whether this intrinsic may mutate heap state and/or throw a -// NullReferenceException if the address is "null". -// -bool GenTreeHWIntrinsic::OperIsMemoryStore(GenTree** pAddr) const -{ - GenTree* addr = nullptr; - -#if defined(TARGET_XARCH) || defined(TARGET_ARM64) - NamedIntrinsic intrinsicId = GetHWIntrinsicId(); - HWIntrinsicCategory category = HWIntrinsicInfo::lookupCategory(intrinsicId); - - if (category == HW_Category_MemoryStore) - { - switch (intrinsicId) - { -#ifdef TARGET_XARCH - case NI_SSE2_MaskMove: - addr = Op(3); - break; - -#elif defined(TARGET_ARM64) - case NI_Sve_StoreAndZip: - case NI_Sve_StoreAndZipx2: - case NI_Sve_StoreAndZipx3: - case NI_Sve_StoreAndZipx4: - case NI_Sve_StoreNarrowing: - case NI_Sve_StoreNonTemporal: - addr = Op(2); - break; - - case NI_Sve_Scatter: - case NI_Sve_Scatter16BitNarrowing: - case NI_Sve_Scatter16BitWithByteOffsetsNarrowing: - case NI_Sve_Scatter32BitNarrowing: - case NI_Sve_Scatter32BitWithByteOffsetsNarrowing: - case NI_Sve_Scatter8BitNarrowing: - case NI_Sve_Scatter8BitWithByteOffsetsNarrowing: - addr = Op(2); - break; -#endif // TARGET_ARM64 - - default: - addr = Op(1); - break; - } - } -#ifdef TARGET_XARCH - else if (HWIntrinsicInfo::MaybeMemoryStore(intrinsicId) && - (category == HW_Category_IMM || category == HW_Category_Scalar)) - { - // Some intrinsics (without HW_Category_MemoryStore) also have MemoryStore semantics - - // Bmi2/Bmi2.X64.MultiplyNoFlags may return the lower half result by a out argument - // unsafe ulong MultiplyNoFlags(ulong left, ulong right, ulong* low) - // - // So, the 3-argument form is MemoryStore - if (GetOperandCount() == 3) - { - switch (intrinsicId) - { - case NI_BMI2_MultiplyNoFlags: - case NI_BMI2_X64_MultiplyNoFlags: - addr = Op(3); - break; - - default: - break; - } - } - } -#endif // TARGET_XARCH -#endif // TARGET_XARCH || TARGET_ARM64 - - if (pAddr != nullptr) - { - *pAddr = addr; - } - - if (addr != nullptr) - { -#ifdef TARGET_ARM64 - assert(varTypeIsI(addr) || (varTypeIsSIMD(addr) && ((intrinsicId >= NI_Sve_Scatter)))); -#else - assert(varTypeIsI(addr)); -#endif - return true; - } - - return false; -} - -//------------------------------------------------------------------------ -// OperIsMemoryLoadOrStore: Does this HWI node have memory load or store semantics? -// -// Return Value: -// Whether "this" is "OperIsMemoryLoad" or "OperIsMemoryStore". -// -bool GenTreeHWIntrinsic::OperIsMemoryLoadOrStore() const -{ - return OperIsMemoryLoad() || OperIsMemoryStore(); -} - -//------------------------------------------------------------------------ -// OperIsMemoryStoreOrBarrier: Does this HWI node have memory store or barrier semantics? -// -// Return Value: -// Whether "this" is a store or memory barrier. -// -bool GenTreeHWIntrinsic::OperIsMemoryStoreOrBarrier() const -{ - if (OperIsMemoryStore()) - { - return true; - } -#if defined(TARGET_XARCH) - NamedIntrinsic intrinsicId = GetHWIntrinsicId(); - - if (HWIntrinsicInfo::HasSpecialSideEffect_Barrier(intrinsicId)) - { - return true; - } -#endif // TARGET_XARCH - else - { - return false; - } -} - -//------------------------------------------------------------------------ -// OperIsEmbBroadcastCompatible: Checks if the intrinsic is a embedded broadcast compatible inintrsic. -// -// Return Value: -// true if the intrinsic node lowering instruction is embedded broadcast compatible. -// -bool GenTreeHWIntrinsic::OperIsEmbBroadcastCompatible() const -{ -#if defined(TARGET_XARCH) - NamedIntrinsic intrinsicId = GetHWIntrinsicId(); - var_types simdBaseType = GetSimdBaseType(); - - // MaybeImm intrinsics support embedded broadcasts only for their IMM variants (e.g. PSLLQ) - if (HWIntrinsicInfo::MaybeImm(intrinsicId) && - !HWIntrinsicInfo::isImmOp(intrinsicId, GetOperandArray()[GetOperandCount() - 1])) - { - return false; - } - - switch (intrinsicId) - { - case NI_AVX512F_ConvertToVector256Int32: - case NI_AVX512F_ConvertToVector256UInt32: - case NI_AVX512F_VL_ConvertToVector128UInt32: - case NI_AVX10v1_ConvertToVector128UInt32: - { - return varTypeIsFloating(simdBaseType); - } - - default: - { - return !varTypeIsSmall(simdBaseType) && HWIntrinsicInfo::IsEmbBroadcastCompatible(intrinsicId); - } - } -#else - return false; -#endif // TARGET_XARCH -} - -//------------------------------------------------------------------------ -// OperIsBroadcastScalar: Is this HWIntrinsic a broadcast node from scalar. -// -// Return Value: -// Whether "this" is a broadcast node from scalar. -// -bool GenTreeHWIntrinsic::OperIsBroadcastScalar() const -{ -#if defined(TARGET_XARCH) - NamedIntrinsic intrinsicId = GetHWIntrinsicId(); - switch (intrinsicId) - { - case NI_AVX2_BroadcastScalarToVector128: - case NI_AVX2_BroadcastScalarToVector256: - case NI_AVX_BroadcastScalarToVector128: - case NI_AVX_BroadcastScalarToVector256: - case NI_SSE3_LoadAndDuplicateToVector128: - case NI_SSE3_MoveAndDuplicate: - case NI_AVX512F_BroadcastScalarToVector512: - return true; - default: - return false; - } -#else - return false; -#endif -} - -//------------------------------------------------------------------------ -// OperIsCreateScalarUnsafe: Is this HWIntrinsic a CreateScalarUnsafe node. -// -// Return Value: -// Whether "this" is a CreateScalarUnsafe node. -// -bool GenTreeHWIntrinsic::OperIsCreateScalarUnsafe() const -{ - NamedIntrinsic intrinsicId = GetHWIntrinsicId(); - - switch (intrinsicId) - { -#if defined(TARGET_ARM64) - case NI_Vector64_CreateScalarUnsafe: -#endif // TARGET_ARM64 - case NI_Vector128_CreateScalarUnsafe: -#if defined(TARGET_XARCH) - case NI_Vector256_CreateScalarUnsafe: - case NI_Vector512_CreateScalarUnsafe: -#endif // TARGET_XARCH - { - return true; - } - - default: - { - return false; - } - } -} - -//------------------------------------------------------------------------ -// OperIsBitwiseHWIntrinsic: Is the operation a bitwise logic operation. -// -// Arguments: -// oper -- The operation to check -// -// Return Value: -// Whether oper is a bitwise logic intrinsic node. -// -bool GenTreeHWIntrinsic::OperIsBitwiseHWIntrinsic(genTreeOps oper) -{ - return (oper == GT_AND) || (oper == GT_AND_NOT) || (oper == GT_NOT) || (oper == GT_OR) || (oper == GT_XOR); -} - -//------------------------------------------------------------------------ -// OperIsBitwiseHWIntrinsic: Is this HWIntrinsic a bitwise logic intrinsic node. -// -// Return Value: -// Whether "this" is a bitwise logic intrinsic node. -// -bool GenTreeHWIntrinsic::OperIsBitwiseHWIntrinsic() const -{ - bool isScalar = false; - genTreeOps oper = GetOperForHWIntrinsicId(&isScalar); - return OperIsBitwiseHWIntrinsic(oper); -} - -//------------------------------------------------------------------------ -// OperIsEmbRoundingEnabled: Is this HWIntrinsic a node with embedded rounding feature. -// -// Return Value: -// Whether "this" is a node with embedded rounding feature. -// -bool GenTreeHWIntrinsic::OperIsEmbRoundingEnabled() const -{ -#if defined(TARGET_XARCH) - NamedIntrinsic intrinsicId = GetHWIntrinsicId(); - - if (!HWIntrinsicInfo::IsEmbRoundingCompatible(intrinsicId)) - { - return false; - } - - size_t numArgs = GetOperandCount(); - switch (intrinsicId) - { - // these intrinsics only have the embedded rounding enabled implementation. - case NI_AVX512F_AddScalar: - case NI_AVX512F_DivideScalar: - case NI_AVX512F_MultiplyScalar: - case NI_AVX512F_SubtractScalar: - case NI_AVX512F_SqrtScalar: - case NI_AVX10v1_AddScalar: - case NI_AVX10v1_DivideScalar: - case NI_AVX10v1_MultiplyScalar: - case NI_AVX10v1_SubtractScalar: - case NI_AVX10v1_SqrtScalar: - { - return true; - } - - case NI_AVX512F_FusedMultiplyAdd: - case NI_AVX512F_FusedMultiplyAddScalar: - case NI_AVX512F_FusedMultiplyAddNegated: - case NI_AVX512F_FusedMultiplyAddNegatedScalar: - case NI_AVX512F_FusedMultiplyAddSubtract: - case NI_AVX512F_FusedMultiplySubtract: - case NI_AVX512F_FusedMultiplySubtractAdd: - case NI_AVX512F_FusedMultiplySubtractNegated: - case NI_AVX512F_FusedMultiplySubtractNegatedScalar: - case NI_AVX512F_FusedMultiplySubtractScalar: - case NI_AVX10v1_FusedMultiplyAddScalar: - case NI_AVX10v1_FusedMultiplyAddNegatedScalar: - case NI_AVX10v1_FusedMultiplySubtractScalar: - case NI_AVX10v1_FusedMultiplySubtractNegatedScalar: - { - return numArgs == 4; - } - - case NI_AVX512F_Add: - case NI_AVX512F_Divide: - case NI_AVX512F_Multiply: - case NI_AVX512F_Subtract: - - case NI_AVX512F_Scale: - case NI_AVX512F_ScaleScalar: - case NI_AVX10v1_ScaleScalar: - - case NI_AVX512F_ConvertScalarToVector128Single: -#if defined(TARGET_AMD64) - case NI_AVX512F_X64_ConvertScalarToVector128Double: - case NI_AVX512F_X64_ConvertScalarToVector128Single: - case NI_AVX10v1_X64_ConvertScalarToVector128Double: - case NI_AVX10v1_X64_ConvertScalarToVector128Single: -#endif // TARGET_AMD64 - case NI_AVX10v1_ConvertScalarToVector128Single: - { - return numArgs == 3; - } - - case NI_AVX512F_Sqrt: - case NI_AVX512F_ConvertToInt32: - case NI_AVX512F_ConvertToUInt32: - case NI_AVX512F_ConvertToVector256Int32: - case NI_AVX512F_ConvertToVector256Single: - case NI_AVX512F_ConvertToVector256UInt32: - case NI_AVX512F_ConvertToVector512Single: - case NI_AVX512F_ConvertToVector512UInt32: - case NI_AVX512F_ConvertToVector512Int32: -#if defined(TARGET_AMD64) - case NI_AVX512F_X64_ConvertToInt64: - case NI_AVX512F_X64_ConvertToUInt64: - case NI_AVX10v1_X64_ConvertToInt64: - case NI_AVX10v1_X64_ConvertToUInt64: -#endif // TARGET_AMD64 - case NI_AVX512DQ_ConvertToVector256Single: - case NI_AVX512DQ_ConvertToVector512Double: - case NI_AVX512DQ_ConvertToVector512Int64: - case NI_AVX512DQ_ConvertToVector512UInt64: - case NI_AVX10v1_ConvertToInt32: - case NI_AVX10v1_ConvertToUInt32: - case NI_AVX10v1_V512_ConvertToVector256Single: - case NI_AVX10v1_V512_ConvertToVector512Double: - case NI_AVX10v1_V512_ConvertToVector512Int64: - case NI_AVX10v1_V512_ConvertToVector512UInt64: - { - return numArgs == 2; - } - - default: - unreached(); - } -#else // !TARGET_XARCH - return false; -#endif // TARGET_XARCH -} - -//------------------------------------------------------------------------------ -// OperRequiresAsgFlag : Check whether the operation requires GTF_ASG flag regardless -// of the children's flags. -// -bool GenTreeHWIntrinsic::OperRequiresAsgFlag() const -{ - // Barriers, while they don't technically do an assignment are modeled the same - // as GT_MEMORYBARRIER which tracks itself as requiring the GTF_ASG flag. - return OperIsMemoryStoreOrBarrier(); -} - -//------------------------------------------------------------------------------ -// OperRequiresCallFlag : Check whether the operation requires GTF_CALL flag regardless -// of the children's flags. -// -bool GenTreeHWIntrinsic::OperRequiresCallFlag() const -{ - NamedIntrinsic intrinsicId = GetHWIntrinsicId(); - - if (HWIntrinsicInfo::HasSpecialSideEffect(intrinsicId)) - { - switch (intrinsicId) - { -#if defined(TARGET_XARCH) - case NI_X86Base_Pause: - case NI_SSE_Prefetch0: - case NI_SSE_Prefetch1: - case NI_SSE_Prefetch2: - case NI_SSE_PrefetchNonTemporal: - { - return true; - } -#endif // TARGET_XARCH - -#if defined(TARGET_ARM64) - case NI_ArmBase_Yield: - case NI_Sve_PrefetchBytes: - case NI_Sve_PrefetchInt16: - case NI_Sve_PrefetchInt32: - case NI_Sve_PrefetchInt64: - case NI_Sve_GatherPrefetch16Bit: - case NI_Sve_GatherPrefetch32Bit: - case NI_Sve_GatherPrefetch64Bit: - case NI_Sve_GatherPrefetch8Bit: - case NI_Sve_GetFfrByte: - case NI_Sve_GetFfrInt16: - case NI_Sve_GetFfrInt32: - case NI_Sve_GetFfrInt64: - case NI_Sve_GetFfrSByte: - case NI_Sve_GetFfrUInt16: - case NI_Sve_GetFfrUInt32: - case NI_Sve_GetFfrUInt64: - case NI_Sve_SetFfr: - { - return true; - } -#endif // TARGET_ARM64 - - default: - { - break; - } - } - } - - return IsUserCall(); -} - -//------------------------------------------------------------------------------ -// OperRequiresGlobRefFlag : Check whether the operation requires GTF_GLOB_REF -// flag regardless of the children's flags. -// -bool GenTreeHWIntrinsic::OperRequiresGlobRefFlag() const -{ - return OperIsMemoryLoad() || OperRequiresAsgFlag() || OperRequiresCallFlag(); -} - -//------------------------------------------------------------------------ -// GetLayout: Get the layout for this TYP_STRUCT HWI node. -// -// Arguments: -// compiler - The Compiler instance -// -// Return Value: -// The struct layout for the node. -// -// Notes: -// Currently, this method synthesizes block layouts, since there is not -// enough space in the GenTreeHWIntrinsic node to store a proper layout -// pointer or number. -// -ClassLayout* GenTreeHWIntrinsic::GetLayout(Compiler* compiler) const -{ - assert(TypeIs(TYP_STRUCT)); - - switch (GetHWIntrinsicId()) - { -#ifdef TARGET_XARCH - case NI_X86Base_DivRem: - return compiler->typGetBlkLayout(genTypeSize(GetSimdBaseType()) * 2); - case NI_X86Base_X64_DivRem: - return compiler->typGetBlkLayout(16); -#endif // TARGET_XARCH -#ifdef TARGET_ARM64 - case NI_AdvSimd_Arm64_LoadPairScalarVector64: - case NI_AdvSimd_Arm64_LoadPairScalarVector64NonTemporal: - case NI_AdvSimd_Arm64_LoadPairVector64: - case NI_AdvSimd_Arm64_LoadPairVector64NonTemporal: - case NI_AdvSimd_Load2xVector64AndUnzip: - case NI_AdvSimd_Load2xVector64: - case NI_AdvSimd_LoadAndInsertScalarVector64x2: - case NI_AdvSimd_LoadAndReplicateToVector64x2: - return compiler->typGetBlkLayout(16); - - case NI_AdvSimd_Arm64_LoadPairVector128: - case NI_AdvSimd_Arm64_LoadPairVector128NonTemporal: - case NI_AdvSimd_Arm64_Load2xVector128AndUnzip: - case NI_AdvSimd_Arm64_Load2xVector128: - case NI_AdvSimd_Load4xVector64: - case NI_AdvSimd_Load4xVector64AndUnzip: - case NI_AdvSimd_LoadAndReplicateToVector64x4: - case NI_AdvSimd_Arm64_LoadAndReplicateToVector128x2: - case NI_AdvSimd_Arm64_LoadAndInsertScalarVector128x2: - case NI_AdvSimd_LoadAndInsertScalarVector64x4: - return compiler->typGetBlkLayout(32); - - case NI_AdvSimd_Load3xVector64AndUnzip: - case NI_AdvSimd_Load3xVector64: - case NI_AdvSimd_LoadAndInsertScalarVector64x3: - case NI_AdvSimd_LoadAndReplicateToVector64x3: - return compiler->typGetBlkLayout(24); - - case NI_AdvSimd_Arm64_Load3xVector128AndUnzip: - case NI_AdvSimd_Arm64_Load3xVector128: - case NI_AdvSimd_Arm64_LoadAndInsertScalarVector128x3: - case NI_AdvSimd_Arm64_LoadAndReplicateToVector128x3: - return compiler->typGetBlkLayout(48); - - case NI_AdvSimd_Arm64_Load4xVector128AndUnzip: - case NI_AdvSimd_Arm64_Load4xVector128: - case NI_AdvSimd_Arm64_LoadAndInsertScalarVector128x4: - case NI_AdvSimd_Arm64_LoadAndReplicateToVector128x4: - return compiler->typGetBlkLayout(64); - - case NI_Sve_Load2xVectorAndUnzip: - return compiler->typGetBlkLayout(compiler->getVectorTByteLength() * 2); - case NI_Sve_Load3xVectorAndUnzip: - return compiler->typGetBlkLayout(compiler->getVectorTByteLength() * 3); - case NI_Sve_Load4xVectorAndUnzip: - return compiler->typGetBlkLayout(compiler->getVectorTByteLength() * 4); - -#endif // TARGET_ARM64 - - default: - unreached(); - } -} - -NamedIntrinsic GenTreeHWIntrinsic::GetHWIntrinsicId() const -{ - NamedIntrinsic id = gtHWIntrinsicId; - int numArgs = HWIntrinsicInfo::lookupNumArgs(id); - bool numArgsUnknown = numArgs < 0; - - assert((static_cast(numArgs) == GetOperandCount()) || numArgsUnknown); - - return id; -} - -void GenTreeHWIntrinsic::SetHWIntrinsicId(NamedIntrinsic intrinsicId) -{ -#ifdef DEBUG - size_t oldOperandCount = GetOperandCount(); - int newOperandCount = HWIntrinsicInfo::lookupNumArgs(intrinsicId); - bool newCountUnknown = newOperandCount < 0; - - // We'll choose to trust the programmer here. - assert((oldOperandCount == static_cast(newOperandCount)) || newCountUnknown); - - // All ids being set must be valid - assert(!HWIntrinsicInfo::IsInvalidNodeId(intrinsicId)); -#endif // DEBUG - - gtHWIntrinsicId = intrinsicId; -} - -/* static */ bool GenTreeHWIntrinsic::Equals(GenTreeHWIntrinsic* op1, GenTreeHWIntrinsic* op2) -{ - return (op1->TypeGet() == op2->TypeGet()) && (op1->GetHWIntrinsicId() == op2->GetHWIntrinsicId()) && - (op1->GetSimdBaseType() == op2->GetSimdBaseType()) && (op1->GetSimdSize() == op2->GetSimdSize()) && - (op1->GetAuxiliaryType() == op2->GetAuxiliaryType()) && OperandsAreEqual(op1, op2); -} - -void GenTreeHWIntrinsic::Initialize(NamedIntrinsic intrinsicId) -{ - SetHWIntrinsicId(intrinsicId); - - if (OperIsMemoryStore()) - { - gtFlags |= (GTF_ASG | GTF_GLOB_REF | GTF_EXCEPT); - } - else if (OperIsMemoryLoad()) - { - gtFlags |= (GTF_GLOB_REF | GTF_EXCEPT); - } - else if (HWIntrinsicInfo::HasSpecialSideEffect(intrinsicId)) - { - switch (intrinsicId) - { -#if defined(TARGET_XARCH) - case NI_SSE_StoreFence: - case NI_SSE2_LoadFence: - case NI_SSE2_MemoryFence: - case NI_X86Serialize_Serialize: - { - // Mark as a store and global reference, much as is done for GT_MEMORYBARRIER - gtFlags |= (GTF_ASG | GTF_GLOB_REF); - break; - } - - case NI_X86Base_Pause: - case NI_SSE_Prefetch0: - case NI_SSE_Prefetch1: - case NI_SSE_Prefetch2: - case NI_SSE_PrefetchNonTemporal: - { - // Mark as a call and global reference, much as is done for GT_KEEPALIVE - gtFlags |= (GTF_CALL | GTF_GLOB_REF); - break; - } -#endif // TARGET_XARCH - -#if defined(TARGET_ARM64) - case NI_ArmBase_Yield: - case NI_Sve_GatherPrefetch16Bit: - case NI_Sve_GatherPrefetch32Bit: - case NI_Sve_GatherPrefetch64Bit: - case NI_Sve_GatherPrefetch8Bit: - case NI_Sve_PrefetchBytes: - case NI_Sve_PrefetchInt16: - case NI_Sve_PrefetchInt32: - case NI_Sve_PrefetchInt64: - case NI_Sve_GetFfrByte: - case NI_Sve_GetFfrInt16: - case NI_Sve_GetFfrInt32: - case NI_Sve_GetFfrInt64: - case NI_Sve_GetFfrSByte: - case NI_Sve_GetFfrUInt16: - case NI_Sve_GetFfrUInt32: - case NI_Sve_GetFfrUInt64: - case NI_Sve_SetFfr: - { - // Mark as a call and global reference, much as is done for GT_KEEPALIVE - gtFlags |= (GTF_CALL | GTF_GLOB_REF); - break; - } -#endif // TARGET_ARM64 - - default: - { - assert(!"Unexpected hwintrinsic side-effect"); - break; - } - } - } -} - -//------------------------------------------------------------------------------ -// GetOperForHWIntrinsicId: Returns oper based on the intrinsic ID and base type -// -// Arguments: -// id - The intrinsic ID for which to get the oper -// simdBaseType - The base type on which id is executed -// isScalar - On return, contains true if the oper is over scalar data; otherwise false -// -// Returns: -// The oper based on the intrinsic ID and base type -// -genTreeOps GenTreeHWIntrinsic::GetOperForHWIntrinsicId(NamedIntrinsic id, var_types simdBaseType, bool* isScalar) -{ - *isScalar = false; - - switch (id) - { -#if defined(TARGET_XARCH) - case NI_SSE_And: - case NI_SSE2_And: - case NI_AVX_And: - case NI_AVX2_And: - case NI_AVX512F_And: - case NI_AVX512DQ_And: - case NI_AVX10v1_V512_And: - case NI_EVEX_AndMask: -#elif defined(TARGET_ARM64) - case NI_AdvSimd_And: -#endif - { - return GT_AND; - } - -#if defined(TARGET_XARCH) - case NI_EVEX_NotMask: -#elif defined(TARGET_ARM64) - case NI_AdvSimd_Not: -#endif - { - return GT_NOT; - } - -#if defined(TARGET_XARCH) - case NI_SSE_Xor: - case NI_SSE2_Xor: - case NI_AVX_Xor: - case NI_AVX2_Xor: - case NI_AVX512F_Xor: - case NI_AVX512DQ_Xor: - case NI_AVX10v1_V512_Xor: - case NI_EVEX_XorMask: -#elif defined(TARGET_ARM64) - case NI_AdvSimd_Xor: -#endif - { - return GT_XOR; - } - -#if defined(TARGET_XARCH) - case NI_SSE_Or: - case NI_SSE2_Or: - case NI_AVX_Or: - case NI_AVX2_Or: - case NI_AVX512F_Or: - case NI_AVX512DQ_Or: - case NI_AVX10v1_V512_Or: - case NI_EVEX_OrMask: -#elif defined(TARGET_ARM64) - case NI_AdvSimd_Or: -#endif - { - return GT_OR; - } - -#if defined(TARGET_XARCH) - case NI_SSE_AndNot: - case NI_SSE2_AndNot: - case NI_AVX_AndNot: - case NI_AVX2_AndNot: - case NI_AVX512F_AndNot: - case NI_AVX512DQ_AndNot: - case NI_AVX10v1_V512_AndNot: - case NI_EVEX_AndNotMask: -#elif defined(TARGET_ARM64) - case NI_AdvSimd_BitwiseClear: -#endif - { - return GT_AND_NOT; - } - -#if defined(TARGET_XARCH) - case NI_SSE_Add: - case NI_SSE2_Add: - case NI_AVX_Add: - case NI_AVX2_Add: - case NI_AVX512F_Add: - case NI_AVX512BW_Add: -#elif defined(TARGET_ARM64) - case NI_AdvSimd_Add: - case NI_AdvSimd_Arm64_Add: -#endif - { - return GT_ADD; - } - -#if defined(TARGET_XARCH) - case NI_SSE_AddScalar: - case NI_SSE2_AddScalar: - case NI_AVX512F_AddScalar: - case NI_AVX10v1_AddScalar: - { - *isScalar = true; - return GT_ADD; - } -#endif - -#if defined(TARGET_ARM64) - case NI_AdvSimd_AddScalar: - { - if (genTypeSize(simdBaseType) != 8) - { - *isScalar = true; - } - return GT_ADD; - } -#endif - -#if defined(TARGET_XARCH) - case NI_SSE_Divide: - case NI_SSE2_Divide: - case NI_AVX_Divide: - case NI_AVX512F_Divide: -#elif defined(TARGET_ARM64) - case NI_AdvSimd_Arm64_Divide: -#endif - { - return GT_DIV; - } - -#if defined(TARGET_XARCH) - case NI_SSE_DivideScalar: - case NI_SSE2_DivideScalar: - case NI_AVX512F_DivideScalar: - case NI_AVX10v1_DivideScalar: - { - *isScalar = true; - return GT_DIV; - } -#endif - -#if defined(TARGET_ARM64) - case NI_AdvSimd_DivideScalar: - { - if (genTypeSize(simdBaseType) != 8) - { - *isScalar = true; - } - return GT_DIV; - } -#endif - -#if defined(TARGET_XARCH) - case NI_SSE_Multiply: - case NI_SSE2_MultiplyLow: - case NI_SSE41_MultiplyLow: - case NI_AVX_Multiply: - case NI_AVX2_MultiplyLow: - case NI_AVX512F_MultiplyLow: - case NI_AVX512BW_MultiplyLow: - case NI_AVX512DQ_MultiplyLow: - case NI_AVX512DQ_VL_MultiplyLow: - case NI_AVX10v1_MultiplyLow: - case NI_AVX10v1_V512_MultiplyLow: -#elif defined(TARGET_ARM64) - case NI_AdvSimd_Multiply: - case NI_AdvSimd_Arm64_Multiply: -#endif - { - return GT_MUL; - } - -#if defined(TARGET_XARCH) - case NI_SSE2_Multiply: - case NI_AVX512F_Multiply: - { - if (varTypeIsFloating(simdBaseType)) - { - return GT_MUL; - } - return GT_NONE; - } -#endif - -#if defined(TARGET_XARCH) - case NI_SSE_MultiplyScalar: - case NI_SSE2_MultiplyScalar: - case NI_AVX512F_MultiplyScalar: - case NI_AVX10v1_MultiplyScalar: - { - *isScalar = true; - return GT_MUL; - } -#endif - -#if defined(TARGET_ARM64) - case NI_AdvSimd_MultiplyScalar: - { - if (genTypeSize(simdBaseType) != 8) - { - *isScalar = true; - } - return GT_MUL; - } -#endif - -#if defined(TARGET_ARM64) - case NI_AdvSimd_Negate: - case NI_AdvSimd_Arm64_Negate: - { - return GT_NEG; - } - - case NI_AdvSimd_NegateScalar: - case NI_AdvSimd_Arm64_NegateScalar: - { - if (genTypeSize(simdBaseType) != 8) - { - *isScalar = true; - } - return GT_NEG; - } -#endif - -#if defined(TARGET_XARCH) - case NI_AVX512F_RotateLeft: - case NI_AVX512F_RotateLeftVariable: - case NI_AVX512F_VL_RotateLeft: - case NI_AVX512F_VL_RotateLeftVariable: - case NI_AVX10v1_RotateLeft: - case NI_AVX10v1_RotateLeftVariable: - { - return GT_ROL; - } - - case NI_AVX512F_RotateRight: - case NI_AVX512F_RotateRightVariable: - case NI_AVX512F_VL_RotateRight: - case NI_AVX512F_VL_RotateRightVariable: - case NI_AVX10v1_RotateRight: - case NI_AVX10v1_RotateRightVariable: - { - return GT_ROR; - } -#endif // TARGET_XARCH - -#if defined(TARGET_XARCH) - case NI_SSE2_ShiftLeftLogical: - case NI_AVX2_ShiftLeftLogical: - case NI_AVX2_ShiftLeftLogicalVariable: - case NI_AVX512F_ShiftLeftLogical: - case NI_AVX512F_ShiftLeftLogicalVariable: - case NI_AVX512BW_ShiftLeftLogical: - case NI_AVX512BW_ShiftLeftLogicalVariable: - case NI_AVX512BW_VL_ShiftLeftLogicalVariable: - case NI_AVX10v1_ShiftLeftLogicalVariable: -#elif defined(TARGET_ARM64) - case NI_AdvSimd_ShiftLeftLogical: -#endif - { - return GT_LSH; - } - -#if defined(TARGET_ARM64) - case NI_AdvSimd_ShiftLeftLogicalScalar: - { - if (genTypeSize(simdBaseType) != 8) - { - *isScalar = true; - } - return GT_LSH; - } -#endif - -#if defined(TARGET_XARCH) - case NI_SSE2_ShiftRightArithmetic: - case NI_AVX2_ShiftRightArithmetic: - case NI_AVX2_ShiftRightArithmeticVariable: - case NI_AVX512F_ShiftRightArithmetic: - case NI_AVX512F_ShiftRightArithmeticVariable: - case NI_AVX512F_VL_ShiftRightArithmetic: - case NI_AVX512F_VL_ShiftRightArithmeticVariable: - case NI_AVX512BW_ShiftRightArithmetic: - case NI_AVX512BW_ShiftRightArithmeticVariable: - case NI_AVX512BW_VL_ShiftRightArithmeticVariable: - case NI_AVX10v1_ShiftRightArithmetic: - case NI_AVX10v1_ShiftRightArithmeticVariable: -#elif defined(TARGET_ARM64) - case NI_AdvSimd_ShiftRightArithmetic: -#endif - { - return GT_RSH; - } - -#if defined(TARGET_ARM64) - case NI_AdvSimd_ShiftRightArithmeticScalar: - { - if (genTypeSize(simdBaseType) != 8) - { - *isScalar = true; - } - return GT_RSH; - } -#endif - -#if defined(TARGET_XARCH) - case NI_SSE2_ShiftRightLogical: - case NI_AVX2_ShiftRightLogical: - case NI_AVX2_ShiftRightLogicalVariable: - case NI_AVX512F_ShiftRightLogical: - case NI_AVX512F_ShiftRightLogicalVariable: - case NI_AVX512BW_ShiftRightLogical: - case NI_AVX512BW_ShiftRightLogicalVariable: - case NI_AVX512BW_VL_ShiftRightLogicalVariable: - case NI_AVX10v1_ShiftRightLogicalVariable: -#elif defined(TARGET_ARM64) - case NI_AdvSimd_ShiftRightLogical: -#endif - { - return GT_RSZ; - } - -#if defined(TARGET_ARM64) - case NI_AdvSimd_ShiftRightLogicalScalar: - { - if (genTypeSize(simdBaseType) != 8) - { - *isScalar = true; - } - return GT_RSZ; - } -#endif - -#if defined(TARGET_XARCH) - case NI_SSE_Subtract: - case NI_SSE2_Subtract: - case NI_AVX_Subtract: - case NI_AVX2_Subtract: - case NI_AVX512F_Subtract: - case NI_AVX512BW_Subtract: -#elif defined(TARGET_ARM64) - case NI_AdvSimd_Subtract: - case NI_AdvSimd_Arm64_Subtract: -#endif - { - return GT_SUB; - } - -#if defined(TARGET_XARCH) - case NI_SSE_SubtractScalar: - case NI_SSE2_SubtractScalar: - case NI_AVX512F_SubtractScalar: - case NI_AVX10v1_SubtractScalar: - { - *isScalar = true; - return GT_SUB; - } -#endif - -#if defined(TARGET_ARM64) - case NI_AdvSimd_SubtractScalar: - { - if (genTypeSize(simdBaseType) != 8) - { - *isScalar = true; - } - return GT_SUB; - } -#endif - -#if defined(TARGET_XARCH) - case NI_SSE_CompareEqual: - case NI_SSE2_CompareEqual: - case NI_SSE41_CompareEqual: - case NI_AVX_CompareEqual: - case NI_AVX2_CompareEqual: - case NI_EVEX_CompareEqualMask: -#elif defined(TARGET_ARM64) - case NI_AdvSimd_CompareEqual: - case NI_AdvSimd_Arm64_CompareEqual: -#endif - { - return GT_EQ; - } - -#if defined(TARGET_XARCH) - case NI_SSE_CompareScalarEqual: - case NI_SSE2_CompareScalarEqual: - { - *isScalar = true; - return GT_EQ; - } -#endif - -#if defined(TARGET_ARM64) - case NI_AdvSimd_Arm64_CompareEqualScalar: - { - if (genTypeSize(simdBaseType) != 8) - { - *isScalar = true; - } - return GT_EQ; - } -#endif - -#if defined(TARGET_XARCH) - case NI_SSE_CompareGreaterThan: - case NI_SSE2_CompareGreaterThan: - case NI_SSE42_CompareGreaterThan: - case NI_AVX_CompareGreaterThan: - case NI_AVX2_CompareGreaterThan: - case NI_EVEX_CompareGreaterThanMask: -#elif defined(TARGET_ARM64) - case NI_AdvSimd_CompareGreaterThan: - case NI_AdvSimd_Arm64_CompareGreaterThan: -#endif - { - return GT_GT; - } - -#if defined(TARGET_XARCH) - case NI_SSE_CompareScalarGreaterThan: - case NI_SSE2_CompareScalarGreaterThan: - { - *isScalar = true; - return GT_GT; - } -#endif - -#if defined(TARGET_ARM64) - case NI_AdvSimd_Arm64_CompareGreaterThanScalar: - { - if (genTypeSize(simdBaseType) != 8) - { - *isScalar = true; - } - return GT_GT; - } -#endif - -#if defined(TARGET_XARCH) - case NI_SSE_CompareGreaterThanOrEqual: - case NI_SSE2_CompareGreaterThanOrEqual: - case NI_AVX_CompareGreaterThanOrEqual: - case NI_EVEX_CompareGreaterThanOrEqualMask: -#elif defined(TARGET_ARM64) - case NI_AdvSimd_CompareGreaterThanOrEqual: - case NI_AdvSimd_Arm64_CompareGreaterThanOrEqual: -#endif - { - return GT_GE; - } - -#if defined(TARGET_XARCH) - case NI_SSE_CompareScalarGreaterThanOrEqual: - case NI_SSE2_CompareScalarGreaterThanOrEqual: - { - *isScalar = true; - return GT_GE; - } -#endif - -#if defined(TARGET_ARM64) - case NI_AdvSimd_Arm64_CompareGreaterThanOrEqualScalar: - { - if (genTypeSize(simdBaseType) != 8) - { - *isScalar = true; - } - return GT_GE; - } -#endif - -#if defined(TARGET_XARCH) - case NI_SSE_CompareLessThan: - case NI_SSE2_CompareLessThan: - case NI_SSE42_CompareLessThan: - case NI_AVX_CompareLessThan: - case NI_AVX2_CompareLessThan: - case NI_EVEX_CompareLessThanMask: -#elif defined(TARGET_ARM64) - case NI_AdvSimd_CompareLessThan: - case NI_AdvSimd_Arm64_CompareLessThan: -#endif - { - return GT_LT; - } - -#if defined(TARGET_XARCH) - case NI_SSE_CompareScalarLessThan: - case NI_SSE2_CompareScalarLessThan: - { - *isScalar = true; - return GT_LT; - } -#endif - -#if defined(TARGET_ARM64) - case NI_AdvSimd_Arm64_CompareLessThanScalar: - { - if (genTypeSize(simdBaseType) != 8) - { - *isScalar = true; - } - return GT_LT; - } -#endif - -#if defined(TARGET_XARCH) - case NI_SSE_CompareLessThanOrEqual: - case NI_SSE2_CompareLessThanOrEqual: - case NI_AVX_CompareLessThanOrEqual: - case NI_EVEX_CompareLessThanOrEqualMask: -#elif defined(TARGET_ARM64) - case NI_AdvSimd_CompareLessThanOrEqual: - case NI_AdvSimd_Arm64_CompareLessThanOrEqual: -#endif - { - return GT_LE; - } - -#if defined(TARGET_XARCH) - case NI_SSE_CompareScalarLessThanOrEqual: - case NI_SSE2_CompareScalarLessThanOrEqual: - { - *isScalar = true; - return GT_LE; - } -#endif - -#if defined(TARGET_ARM64) - case NI_AdvSimd_Arm64_CompareLessThanOrEqualScalar: - { - if (genTypeSize(simdBaseType) != 8) - { - *isScalar = true; - } - return GT_LE; - } -#endif - -#if defined(TARGET_XARCH) - case NI_SSE_CompareNotEqual: - case NI_SSE2_CompareNotEqual: - case NI_AVX_CompareNotEqual: - case NI_EVEX_CompareNotEqualMask: - { - return GT_NE; - } - - case NI_SSE_CompareScalarNotEqual: - case NI_SSE2_CompareScalarNotEqual: - { - *isScalar = true; - return GT_NE; - } -#endif - - default: - { - return GT_NONE; - } - } -} - -//------------------------------------------------------------------------------ -// GetHWIntrinsicIdForUnOp: Returns intrinsic ID based on the oper, base type, and simd size -// -// Arguments: -// comp - The compiler instance -// oper - The oper for which to get the intrinsic ID -// op1 - The first operand on which oper is executed -// simdBaseType - The base type on which oper is executed -// simdSize - The simd size on which oper is executed -// isScalar - True if the oper is over scalar data; otherwise false -// -// Returns: -// The intrinsic ID based on the oper, base type, and simd size -// -NamedIntrinsic GenTreeHWIntrinsic::GetHWIntrinsicIdForUnOp( - Compiler* comp, genTreeOps oper, GenTree* op1, var_types simdBaseType, unsigned simdSize, bool isScalar) -{ - var_types simdType = comp->getSIMDTypeForSize(simdSize); - - assert(varTypeIsArithmetic(simdBaseType)); - assert(varTypeIsSIMD(simdType)); - - if (simdSize == 64) - { - assert(!isScalar); - assert(comp->IsBaselineVector512IsaSupportedDebugOnly()); - } - else if (simdSize == 32) - { - assert(!isScalar); - assert(comp->IsBaselineVector256IsaSupportedDebugOnly()); - } - else - { -#if defined(TARGET_ARM64) - assert(!isScalar || (simdSize == 8)); -#endif // TARGET_ARM64 - - assert(!isScalar || varTypeIsFloating(simdBaseType)); - assert(comp->IsBaselineSimdIsaSupportedDebugOnly()); - } - - assert(op1 != nullptr); - assert(op1->TypeIs(simdType)); - - NamedIntrinsic id = NI_Illegal; - - switch (oper) - { - case GT_NEG: - { -#if defined(TARGET_ARM64) - assert(varTypeIsSigned(simdBaseType) || varTypeIsFloating(simdBaseType)); - - if (varTypeIsLong(simdBaseType)) - { - id = (simdSize == 8) ? NI_AdvSimd_Arm64_NegateScalar : NI_AdvSimd_Arm64_Negate; - } - else if ((simdSize == 8) && (isScalar || (genTypeSize(simdBaseType) == 8))) - { - id = NI_AdvSimd_NegateScalar; - } - else if (simdBaseType == TYP_DOUBLE) - { - id = NI_AdvSimd_Arm64_Negate; - } - else - { - id = NI_AdvSimd_Negate; - } -#endif // TARGET_ARM64 - break; - } - - case GT_NOT: - { - assert(!isScalar); - -#if defined(TARGET_ARM64) - id = NI_AdvSimd_Not; -#endif // TARGET_ARM64 - break; - } - - default: - { - unreached(); - } - } - - return id; -} - -//------------------------------------------------------------------------------ -// GetHWIntrinsicIdForBinOp: Returns intrinsic ID based on the oper, base type, and simd size -// -// Arguments: -// comp - The compiler instance -// oper - The oper for which to get the intrinsic ID -// op1 - The first operand on which oper is executed -// op2 - The second operand on which oper is executed -// simdBaseType - The base type on which oper is executed -// simdSize - The simd size on which oper is executed -// isScalar - True if the oper is over scalar data; otherwise false -// -// Returns: -// The intrinsic ID based on the oper, base type, and simd size -// -NamedIntrinsic GenTreeHWIntrinsic::GetHWIntrinsicIdForBinOp(Compiler* comp, - genTreeOps oper, - GenTree* op1, - GenTree* op2, - var_types simdBaseType, - unsigned simdSize, - bool isScalar) -{ - var_types simdType = comp->getSIMDTypeForSize(simdSize); - - assert(varTypeIsArithmetic(simdBaseType)); - assert(varTypeIsSIMD(simdType)); - - assert(op1 != nullptr); - assert(op1->TypeIs(simdType)); - assert(op2 != nullptr); - - if (simdSize == 64) - { - assert(!isScalar); - assert(comp->IsBaselineVector512IsaSupportedDebugOnly()); - } - else if (simdSize == 32) - { - assert(!isScalar); - assert(comp->IsBaselineVector256IsaSupportedDebugOnly()); - } - else - { -#if defined(TARGET_ARM64) - assert(!isScalar || (simdSize == 8)); -#endif // TARGET_ARM64 - - assert(!isScalar || varTypeIsFloating(simdBaseType)); - assert(comp->IsBaselineSimdIsaSupportedDebugOnly()); - } - - NamedIntrinsic id = NI_Illegal; - - switch (oper) - { - case GT_ADD: - { - assert(op2->TypeIs(simdType)); - -#if defined(TARGET_XARCH) - if (simdSize == 64) - { - if (varTypeIsSmall(simdBaseType)) - { - id = NI_AVX512BW_Add; - } - else - { - id = NI_AVX512F_Add; - } - } - else if (simdSize == 32) - { - if (varTypeIsIntegral(simdBaseType)) - { - assert(comp->compIsaSupportedDebugOnly(InstructionSet_AVX2)); - id = NI_AVX2_Add; - } - else - { - id = NI_AVX_Add; - } - } - else if (simdBaseType == TYP_FLOAT) - { - id = isScalar ? NI_SSE_AddScalar : NI_SSE_Add; - } - else - { - assert(comp->compIsaSupportedDebugOnly(InstructionSet_SSE2)); - id = isScalar ? NI_SSE2_AddScalar : NI_SSE2_Add; - } -#elif defined(TARGET_ARM64) - if ((simdSize == 8) && (isScalar || (genTypeSize(simdBaseType) == 8))) - { - id = NI_AdvSimd_AddScalar; - } - else if (simdBaseType == TYP_DOUBLE) - { - id = NI_AdvSimd_Arm64_Add; - } - else - { - id = NI_AdvSimd_Add; - } -#endif // !TARGET_XARCH && !TARGET_ARM64 - break; - } - - case GT_AND: - { - assert(!isScalar); - assert(op2->TypeIs(simdType)); - -#if defined(TARGET_XARCH) - if (simdSize == 64) - { - if (varTypeIsFloating(simdBaseType)) - { - id = NI_AVX512DQ_And; - } - else - { - id = NI_AVX512F_And; - } - } - else if (simdSize == 32) - { - if (varTypeIsIntegral(simdBaseType)) - { - assert(comp->compIsaSupportedDebugOnly(InstructionSet_AVX2)); - id = NI_AVX2_And; - } - else - { - id = NI_AVX_And; - } - } - else if (simdBaseType == TYP_FLOAT) - { - id = NI_SSE_And; - } - else - { - assert(comp->compIsaSupportedDebugOnly(InstructionSet_SSE2)); - id = NI_SSE2_And; - } -#elif defined(TARGET_ARM64) - id = NI_AdvSimd_And; -#endif // !TARGET_XARCH && !TARGET_ARM64 - break; - } - - case GT_AND_NOT: - { - assert(!isScalar); - assert(op2->TypeIs(simdType)); - - if (comp->fgNodeThreading != NodeThreading::LIR) - { - // We don't want to support creating AND_NOT nodes prior to LIR - // as it can break important optimizations. We'll produces this - // in lowering instead. - break; - } - -#if defined(TARGET_XARCH) - if (simdSize == 64) - { - if (varTypeIsFloating(simdBaseType)) - { - id = NI_AVX512DQ_AndNot; - } - else - { - id = NI_AVX512F_AndNot; - } - } - else if (simdSize == 32) - { - if (varTypeIsIntegral(simdBaseType)) - { - assert(comp->compIsaSupportedDebugOnly(InstructionSet_AVX2)); - id = NI_AVX2_AndNot; - } - else - { - id = NI_AVX_AndNot; - } - } - else if (simdBaseType == TYP_FLOAT) - { - id = NI_SSE_AndNot; - } - else - { - assert(comp->compIsaSupportedDebugOnly(InstructionSet_SSE2)); - id = NI_SSE2_AndNot; - } -#elif defined(TARGET_ARM64) - - id = NI_AdvSimd_BitwiseClear; -#endif // !TARGET_XARCH && !TARGET_ARM64 - break; - } - - case GT_DIV: - { - assert(varTypeIsFloating(simdBaseType)); - assert(op2->TypeIs(simdType)); - -#if defined(TARGET_XARCH) - if (simdSize == 64) - { - id = NI_AVX512F_Divide; - } - else if (simdSize == 32) - { - id = NI_AVX_Divide; - } - else if (simdBaseType == TYP_FLOAT) - { - id = isScalar ? NI_SSE_DivideScalar : NI_SSE_Divide; - } - else - { - assert(comp->compIsaSupportedDebugOnly(InstructionSet_SSE2)); - id = isScalar ? NI_SSE2_DivideScalar : NI_SSE2_Divide; - } -#elif defined(TARGET_ARM64) - if ((simdSize == 8) && (isScalar || (simdBaseType == TYP_DOUBLE))) - { - id = NI_AdvSimd_DivideScalar; - } - else - { - id = NI_AdvSimd_Arm64_Divide; - } -#endif // !TARGET_XARCH && !TARGET_ARM64 - break; - } - - case GT_LSH: - { - assert(!isScalar); - assert(op2->TypeIs(simdType) || varTypeIsInt(op2)); - assert(varTypeIsIntegral(simdBaseType)); - -#if defined(TARGET_XARCH) - if (simdSize == 64) - { - if (varTypeIsShort(simdBaseType)) - { - id = varTypeIsInt(op2) ? NI_AVX512BW_ShiftLeftLogical : NI_AVX512BW_ShiftLeftLogicalVariable; - } - else if (!varTypeIsByte(simdBaseType)) - { - id = varTypeIsInt(op2) ? NI_AVX512F_ShiftLeftLogical : NI_AVX512F_ShiftLeftLogicalVariable; - } - } - else if (varTypeIsShort(simdBaseType)) - { - if (varTypeIsInt(op2)) - { - if (simdSize == 32) - { - assert(comp->compIsaSupportedDebugOnly(InstructionSet_AVX2)); - id = NI_AVX2_ShiftLeftLogical; - } - else - { - assert(comp->compIsaSupportedDebugOnly(InstructionSet_SSE2)); - id = NI_SSE2_ShiftLeftLogical; - } - } - else - { - bool isV512Supported = false; - if (comp->compIsEvexOpportunisticallySupported(isV512Supported, InstructionSet_AVX512BW_VL)) - { - id = isV512Supported ? NI_AVX512BW_VL_ShiftLeftLogicalVariable - : NI_AVX10v1_ShiftLeftLogicalVariable; - } - } - } - else if (!varTypeIsByte(simdBaseType)) - { - if (simdSize == 32) - { - assert(comp->compIsaSupportedDebugOnly(InstructionSet_AVX2)); - id = varTypeIsInt(op2) ? NI_AVX2_ShiftLeftLogical : NI_AVX2_ShiftLeftLogicalVariable; - } - else if (varTypeIsInt(op2)) - { - assert(comp->compIsaSupportedDebugOnly(InstructionSet_SSE2)); - id = NI_SSE2_ShiftLeftLogical; - } - else if (comp->compOpportunisticallyDependsOn(InstructionSet_AVX2)) - { - id = NI_AVX2_ShiftLeftLogicalVariable; - } - } -#elif defined(TARGET_ARM64) - if ((simdSize == 8) && (genTypeSize(simdBaseType) == 8)) - { - id = op2->IsCnsIntOrI() ? NI_AdvSimd_ShiftLeftLogicalScalar : NI_AdvSimd_ShiftLogicalScalar; - } - else - { - id = op2->IsCnsIntOrI() ? NI_AdvSimd_ShiftLeftLogical : NI_AdvSimd_ShiftLogical; - } -#endif // !TARGET_XARCH && !TARGET_ARM64 - break; - } - - case GT_MUL: - { -#if defined(TARGET_XARCH) - assert(op2->TypeIs(simdType)); - - if (simdSize == 64) - { - if (varTypeIsFloating(simdBaseType)) - { - id = NI_AVX512F_Multiply; - } - else if (varTypeIsLong(simdBaseType)) - { - id = NI_AVX512DQ_MultiplyLow; - } - else if (varTypeIsInt(simdBaseType)) - { - id = NI_AVX512F_MultiplyLow; - } - else if (varTypeIsShort(simdBaseType)) - { - id = NI_AVX512BW_MultiplyLow; - } - } - else if (varTypeIsLong(simdBaseType)) - { - bool isV512Supported = false; - if (comp->compIsEvexOpportunisticallySupported(isV512Supported, InstructionSet_AVX512DQ_VL)) - { - id = isV512Supported ? NI_AVX512DQ_VL_MultiplyLow : NI_AVX10v1_MultiplyLow; - } - } - else if (simdSize == 32) - { - if (varTypeIsFloating(simdBaseType)) - { - id = NI_AVX_Multiply; - } - else if (!varTypeIsByte(simdBaseType)) - { - assert(comp->compIsaSupportedDebugOnly(InstructionSet_AVX2)); - id = NI_AVX2_MultiplyLow; - } - } - else if (simdBaseType == TYP_FLOAT) - { - id = isScalar ? NI_SSE_MultiplyScalar : NI_SSE_Multiply; - } - else if (simdBaseType == TYP_DOUBLE) - { - assert(comp->compIsaSupportedDebugOnly(InstructionSet_SSE2)); - id = isScalar ? NI_SSE2_MultiplyScalar : NI_SSE2_Multiply; - } - else if (varTypeIsInt(simdBaseType)) - { - if (comp->compOpportunisticallyDependsOn(InstructionSet_SSE41)) - { - id = NI_SSE41_MultiplyLow; - } - } - else if (varTypeIsShort(simdBaseType)) - { - assert(comp->compIsaSupportedDebugOnly(InstructionSet_SSE2)); - id = NI_SSE2_MultiplyLow; - } -#elif defined(TARGET_ARM64) - if ((simdSize == 8) && (isScalar || (simdBaseType == TYP_DOUBLE))) - { - id = NI_AdvSimd_MultiplyScalar; - } - else if (simdBaseType == TYP_DOUBLE) - { - id = op2->TypeIs(simdType) ? NI_AdvSimd_Arm64_Multiply : NI_AdvSimd_Arm64_MultiplyByScalar; - } - else if (!varTypeIsLong(simdBaseType)) - { - id = op2->TypeIs(simdType) ? NI_AdvSimd_Multiply : NI_AdvSimd_MultiplyByScalar; - } -#endif // !TARGET_XARCH && !TARGET_ARM64 - break; - } - - case GT_OR: - { - assert(!isScalar); - assert(op2->TypeIs(simdType)); - -#if defined(TARGET_XARCH) - if (simdSize == 64) - { - if (varTypeIsFloating(simdBaseType)) - { - id = NI_AVX512DQ_Or; - } - else - { - id = NI_AVX512F_Or; - } - } - else if (simdSize == 32) - { - if (varTypeIsIntegral(simdBaseType)) - { - assert(comp->compIsaSupportedDebugOnly(InstructionSet_AVX2)); - id = NI_AVX2_Or; - } - else - { - id = NI_AVX_Or; - } - } - else if (simdBaseType == TYP_FLOAT) - { - id = NI_SSE_Or; - } - else - { - assert(comp->compIsaSupportedDebugOnly(InstructionSet_SSE2)); - id = NI_SSE2_Or; - } -#elif defined(TARGET_ARM64) - - id = NI_AdvSimd_Or; -#endif // !TARGET_XARCH && !TARGET_ARM64 - break; - } - - case GT_ROL: - { - assert(!isScalar); - assert(op2->TypeIs(simdType) || varTypeIsInt(op2)); - assert(varTypeIsIntegral(simdBaseType)); - -#if defined(TARGET_XARCH) - if (simdSize == 64) - { - if (!varTypeIsSmall(simdBaseType)) - { - id = varTypeIsInt(op2) ? NI_AVX512F_RotateLeft : NI_AVX512F_RotateLeftVariable; - } - } - else if (!varTypeIsSmall(simdBaseType)) - { - bool isV512Supported = false; - if (comp->compIsEvexOpportunisticallySupported(isV512Supported, InstructionSet_AVX512DQ_VL)) - { - if (isV512Supported) - { - id = varTypeIsInt(op2) ? NI_AVX512F_VL_RotateLeft : NI_AVX512F_VL_RotateLeftVariable; - } - else - { - id = varTypeIsInt(op2) ? NI_AVX10v1_RotateLeft : NI_AVX10v1_RotateLeftVariable; - } - } - } -#endif // TARGET_XARCH - break; - } - - case GT_ROR: - { - assert(!isScalar); - assert(op2->TypeIs(simdType) || varTypeIsInt(op2)); - assert(varTypeIsIntegral(simdBaseType)); - -#if defined(TARGET_XARCH) - if (simdSize == 64) - { - if (!varTypeIsSmall(simdBaseType)) - { - id = varTypeIsInt(op2) ? NI_AVX512F_RotateRight : NI_AVX512F_RotateRightVariable; - } - } - else if (!varTypeIsSmall(simdBaseType)) - { - bool isV512Supported = false; - if (comp->compIsEvexOpportunisticallySupported(isV512Supported, InstructionSet_AVX512DQ_VL)) - { - if (isV512Supported) - { - id = varTypeIsInt(op2) ? NI_AVX512F_VL_RotateRight : NI_AVX512F_VL_RotateRightVariable; - } - else - { - id = varTypeIsInt(op2) ? NI_AVX10v1_RotateRight : NI_AVX10v1_RotateRightVariable; - } - } - } -#endif // TARGET_XARCH - break; - } - - case GT_RSH: - { - assert(!isScalar); - assert(op2->TypeIs(simdType) || varTypeIsInt(op2)); - assert(varTypeIsIntegral(simdBaseType)); - -#if defined(TARGET_XARCH) - if (simdSize == 64) - { - if (varTypeIsShort(simdBaseType)) - { - id = - varTypeIsInt(op2) ? NI_AVX512BW_ShiftRightArithmetic : NI_AVX512BW_ShiftRightArithmeticVariable; - } - else if (!varTypeIsByte(simdBaseType)) - { - id = varTypeIsInt(op2) ? NI_AVX512F_ShiftRightArithmetic : NI_AVX512F_ShiftRightArithmeticVariable; - } - } - else if (genTypeSize(simdBaseType) == 8) - { - bool isV512Supported = false; - if (comp->compIsEvexOpportunisticallySupported(isV512Supported, InstructionSet_AVX512F_VL)) - { - if (isV512Supported) - { - id = varTypeIsInt(op2) ? NI_AVX512F_VL_ShiftRightArithmetic - : NI_AVX512F_VL_ShiftRightArithmeticVariable; - } - else - { - id = varTypeIsInt(op2) ? NI_AVX10v1_ShiftRightArithmetic - : NI_AVX10v1_ShiftRightArithmeticVariable; - } - } - } - else if (varTypeIsShort(simdBaseType)) - { - if (varTypeIsInt(op2)) - { - if (simdSize == 32) - { - assert(comp->compIsaSupportedDebugOnly(InstructionSet_AVX2)); - id = NI_AVX2_ShiftRightArithmetic; - } - else - { - assert(comp->compIsaSupportedDebugOnly(InstructionSet_SSE2)); - id = NI_SSE2_ShiftRightArithmetic; - } - } - else - { - bool isV512Supported = false; - if (comp->compIsEvexOpportunisticallySupported(isV512Supported, InstructionSet_AVX512BW_VL)) - { - id = isV512Supported ? NI_AVX512BW_VL_ShiftRightArithmeticVariable - : NI_AVX10v1_ShiftRightArithmeticVariable; - } - } - } - else if (!varTypeIsByte(simdBaseType)) - { - if (simdSize == 32) - { - assert(comp->compIsaSupportedDebugOnly(InstructionSet_AVX2)); - id = varTypeIsInt(op2) ? NI_AVX2_ShiftRightArithmetic : NI_AVX2_ShiftRightArithmeticVariable; - } - else if (varTypeIsInt(op2)) - { - assert(comp->compIsaSupportedDebugOnly(InstructionSet_SSE2)); - id = NI_SSE2_ShiftRightArithmetic; - } - else if (comp->compOpportunisticallyDependsOn(InstructionSet_AVX2)) - { - id = NI_AVX2_ShiftRightArithmeticVariable; - } - } -#elif defined(TARGET_ARM64) - if ((simdSize == 8) && (genTypeSize(simdBaseType) == 8)) - { - id = op2->IsCnsIntOrI() ? NI_AdvSimd_ShiftRightArithmeticScalar : NI_AdvSimd_ShiftArithmeticScalar; - } - else - { - id = op2->IsCnsIntOrI() ? NI_AdvSimd_ShiftRightArithmetic : NI_AdvSimd_ShiftArithmetic; - } -#endif // !TARGET_XARCH && !TARGET_ARM64 - break; - } - - case GT_RSZ: - { - assert(!isScalar); - assert(op2->TypeIs(simdType) || varTypeIsInt(op2)); - assert(varTypeIsIntegral(simdBaseType)); - -#if defined(TARGET_XARCH) - if (simdSize == 64) - { - if (varTypeIsShort(simdBaseType)) - { - id = varTypeIsInt(op2) ? NI_AVX512BW_ShiftRightLogical : NI_AVX512BW_ShiftRightLogicalVariable; - } - else if (!varTypeIsByte(simdBaseType)) - { - id = varTypeIsInt(op2) ? NI_AVX512F_ShiftRightLogical : NI_AVX512F_ShiftRightLogicalVariable; - } - } - else if (varTypeIsShort(simdBaseType)) - { - if (varTypeIsInt(op2)) - { - if (simdSize == 32) - { - assert(comp->compIsaSupportedDebugOnly(InstructionSet_AVX2)); - id = NI_AVX2_ShiftRightLogical; - } - else - { - assert(comp->compIsaSupportedDebugOnly(InstructionSet_SSE2)); - id = NI_SSE2_ShiftRightLogical; - } - } - else - { - bool isV512Supported = false; - if (comp->compIsEvexOpportunisticallySupported(isV512Supported, InstructionSet_AVX512BW_VL)) - { - id = isV512Supported ? NI_AVX512BW_VL_ShiftRightLogicalVariable - : NI_AVX10v1_ShiftRightLogicalVariable; - } - } - } - else if (!varTypeIsByte(simdBaseType)) - { - if (simdSize == 32) - { - assert(comp->compIsaSupportedDebugOnly(InstructionSet_AVX2)); - id = varTypeIsInt(op2) ? NI_AVX2_ShiftRightLogical : NI_AVX2_ShiftRightLogicalVariable; - } - else if (varTypeIsInt(op2)) - { - assert(comp->compIsaSupportedDebugOnly(InstructionSet_SSE2)); - id = NI_SSE2_ShiftRightLogical; - } - else if (comp->compOpportunisticallyDependsOn(InstructionSet_AVX2)) - { - id = NI_AVX2_ShiftRightLogicalVariable; - } - } -#elif defined(TARGET_ARM64) - if ((simdSize == 8) && (genTypeSize(simdBaseType) == 8)) - { - id = varTypeIsInt(op2) ? NI_AdvSimd_ShiftRightLogicalScalar : NI_AdvSimd_ShiftLogicalScalar; - } - else - { - id = varTypeIsInt(op2) ? NI_AdvSimd_ShiftRightLogical : NI_AdvSimd_ShiftLogical; - } -#endif // !TARGET_XARCH && !TARGET_ARM64 - break; - } - - case GT_SUB: - { - assert(op2->TypeIs(simdType)); - -#if defined(TARGET_XARCH) - if (simdSize == 64) - { - if (varTypeIsSmall(simdBaseType)) - { - id = NI_AVX512BW_Subtract; - } - else - { - id = NI_AVX512F_Subtract; - } - } - else if (simdSize == 32) - { - if (varTypeIsIntegral(simdBaseType)) - { - assert(comp->compIsaSupportedDebugOnly(InstructionSet_AVX2)); - id = NI_AVX2_Subtract; - } - else - { - id = NI_AVX_Subtract; - } - } - else if (simdBaseType == TYP_FLOAT) - { - id = isScalar ? NI_SSE_SubtractScalar : NI_SSE_Subtract; - } - else - { - assert(comp->compIsaSupportedDebugOnly(InstructionSet_SSE2)); - id = isScalar ? NI_SSE2_SubtractScalar : NI_SSE2_Subtract; - } -#elif defined(TARGET_ARM64) - if ((simdSize == 8) && (isScalar || (genTypeSize(simdBaseType) == 8))) - { - id = NI_AdvSimd_SubtractScalar; - } - else if (simdBaseType == TYP_DOUBLE) - { - id = NI_AdvSimd_Arm64_Subtract; - } - else - { - id = NI_AdvSimd_Subtract; - } -#endif // !TARGET_XARCH && !TARGET_ARM64 - break; - } - - case GT_XOR: - { - assert(!isScalar); - assert(op2->TypeIs(simdType)); - -#if defined(TARGET_XARCH) - if (simdSize == 64) - { - if (varTypeIsFloating(simdBaseType)) - { - id = NI_AVX512DQ_Xor; - } - else - { - id = NI_AVX512F_Xor; - } - } - else if (simdSize == 32) - { - if (varTypeIsIntegral(simdBaseType)) - { - assert(comp->compIsaSupportedDebugOnly(InstructionSet_AVX2)); - id = NI_AVX2_Xor; - } - else - { - id = NI_AVX_Xor; - } - } - else if (simdBaseType == TYP_FLOAT) - { - id = NI_SSE_Xor; - } - else - { - assert(comp->compIsaSupportedDebugOnly(InstructionSet_SSE2)); - id = NI_SSE2_Xor; - } -#elif defined(TARGET_ARM64) - id = NI_AdvSimd_Xor; -#endif // !TARGET_XARCH && !TARGET_ARM64 - break; - } - - default: - { - unreached(); - } - } - - return id; -} - -//------------------------------------------------------------------------------ -// GetHWIntrinsicIdForCmpOp: Returns intrinsic ID based on the oper, base type, and simd size -// -// Arguments: -// comp - The compiler instance -// oper - The oper for which to get the intrinsic ID -// type - The return type of the comparison -// op1 - The first operand on which oper is executed -// op2 - The second operand on which oper is executed -// simdBaseType - The base type on which oper is executed -// simdSize - The simd size on which oper is executed -// isScalar - True if the oper is over scalar data; otherwise false -// -// Returns: -// The intrinsic ID based on the oper, base type, and simd size -// -NamedIntrinsic GenTreeHWIntrinsic::GetHWIntrinsicIdForCmpOp(Compiler* comp, - genTreeOps oper, - var_types type, - GenTree* op1, - GenTree* op2, - var_types simdBaseType, - unsigned simdSize, - bool isScalar) -{ - var_types simdType = comp->getSIMDTypeForSize(simdSize); - assert(varTypeIsMask(type) || (type == simdType)); - - assert(varTypeIsArithmetic(simdBaseType)); - assert(varTypeIsSIMD(simdType)); - - assert(op1 != nullptr); - assert(op1->TypeIs(simdType)); - assert(op2 != nullptr); - - if (varTypeIsMask(type)) - { - assert(!isScalar); - assert(comp->canUseEvexEncodingDebugOnly()); - } - else if (simdSize == 32) - { - assert(!isScalar); - assert(comp->IsBaselineVector256IsaSupportedDebugOnly()); - } - else - { - assert((simdSize == 8) || (simdSize == 12) || (simdSize == 16)); - -#if defined(TARGET_ARM64) - assert(!isScalar || (simdSize == 8)); -#endif // TARGET_ARM64 - - assert(!isScalar || varTypeIsFloating(simdBaseType)); - assert(comp->IsBaselineSimdIsaSupportedDebugOnly()); - } - - NamedIntrinsic id = NI_Illegal; - - switch (oper) - { - case GT_EQ: - { - assert(op2->TypeIs(simdType)); - -#if defined(TARGET_XARCH) - if (varTypeIsMask(type)) - { - id = NI_EVEX_CompareEqualMask; - } - else if (simdSize == 32) - { - if (varTypeIsIntegral(simdBaseType)) - { - assert(comp->compIsaSupportedDebugOnly(InstructionSet_AVX2)); - id = NI_AVX2_CompareEqual; - } - else - { - id = NI_AVX_CompareEqual; - } - } - else if (simdBaseType == TYP_FLOAT) - { - id = isScalar ? NI_SSE_CompareScalarEqual : NI_SSE_CompareEqual; - } - else if (varTypeIsLong(simdBaseType)) - { - if (comp->compOpportunisticallyDependsOn(InstructionSet_SSE41)) - { - id = NI_SSE41_CompareEqual; - } - } - else - { - assert(comp->compIsaSupportedDebugOnly(InstructionSet_SSE2)); - id = isScalar ? NI_SSE2_CompareScalarEqual : NI_SSE2_CompareEqual; - } -#elif defined(TARGET_ARM64) - if (genTypeSize(simdBaseType) == 8) - { - id = (simdSize == 8) ? NI_AdvSimd_Arm64_CompareEqualScalar : NI_AdvSimd_Arm64_CompareEqual; - } - else - { - id = NI_AdvSimd_CompareEqual; - } -#endif // !TARGET_XARCH && !TARGET_ARM64 - break; - } - - case GT_GE: - { - assert(op2->TypeIs(simdType)); - -#if defined(TARGET_XARCH) - if (varTypeIsMask(type)) - { - id = NI_EVEX_CompareGreaterThanOrEqualMask; - } - else if (varTypeIsIntegral(simdBaseType)) - { - // This should have been handled by the caller setting type = TYP_MASK - assert(!comp->canUseEvexEncodingDebugOnly()); - } - else if (simdSize == 32) - { - id = NI_AVX_CompareGreaterThanOrEqual; - } - else if (simdBaseType == TYP_FLOAT) - { - id = isScalar ? NI_SSE_CompareScalarGreaterThanOrEqual : NI_SSE_CompareGreaterThanOrEqual; - } - else - { - assert(comp->compIsaSupportedDebugOnly(InstructionSet_SSE2)); - id = isScalar ? NI_SSE2_CompareScalarGreaterThanOrEqual : NI_SSE2_CompareGreaterThanOrEqual; - } -#elif defined(TARGET_ARM64) - if (genTypeSize(simdBaseType) == 8) - { - id = (simdSize == 8) ? NI_AdvSimd_Arm64_CompareGreaterThanOrEqualScalar - : NI_AdvSimd_Arm64_CompareGreaterThanOrEqual; - } - else - { - id = NI_AdvSimd_CompareGreaterThanOrEqual; - } -#endif // !TARGET_XARCH && !TARGET_ARM64 - break; - } - - case GT_GT: - { - assert(op2->TypeIs(simdType)); - -#if defined(TARGET_XARCH) - if (varTypeIsMask(type)) - { - id = NI_EVEX_CompareGreaterThanMask; - } - else if (varTypeIsIntegral(simdBaseType)) - { - if (varTypeIsSigned(simdBaseType)) - { - if (simdSize == 32) - { - assert(comp->compIsaSupportedDebugOnly(InstructionSet_AVX2)); - id = NI_AVX2_CompareGreaterThan; - } - else if (varTypeIsLong(simdBaseType)) - { - if (comp->compOpportunisticallyDependsOn(InstructionSet_SSE42)) - { - id = NI_SSE42_CompareGreaterThan; - } - } - else - { - assert(comp->compIsaSupportedDebugOnly(InstructionSet_SSE2)); - id = NI_SSE2_CompareGreaterThan; - } - } - else - { - // This should have been handled by the caller setting type = TYP_MASK - assert(!comp->canUseEvexEncodingDebugOnly()); - } - } - else if (simdSize == 32) - { - id = NI_AVX_CompareGreaterThan; - } - else if (simdBaseType == TYP_FLOAT) - { - id = isScalar ? NI_SSE_CompareScalarGreaterThan : NI_SSE_CompareGreaterThan; - } - else - { - assert(comp->compIsaSupportedDebugOnly(InstructionSet_SSE2)); - id = isScalar ? NI_SSE2_CompareScalarGreaterThan : NI_SSE2_CompareGreaterThan; - } -#elif defined(TARGET_ARM64) - if (genTypeSize(simdBaseType) == 8) - { - id = (simdSize == 8) ? NI_AdvSimd_Arm64_CompareGreaterThanScalar : NI_AdvSimd_Arm64_CompareGreaterThan; - } - else - { - id = NI_AdvSimd_CompareGreaterThan; - } -#endif // !TARGET_XARCH && !TARGET_ARM64 - break; - } - - case GT_LE: - { - assert(op2->TypeIs(simdType)); - -#if defined(TARGET_XARCH) - if (varTypeIsMask(type)) - { - id = NI_EVEX_CompareLessThanOrEqualMask; - } - else if (varTypeIsIntegral(simdBaseType)) - { - // This should have been handled by the caller setting type = TYP_MASK - assert(!comp->canUseEvexEncodingDebugOnly()); - } - else if (simdSize == 32) - { - id = NI_AVX_CompareLessThanOrEqual; - } - else if (simdBaseType == TYP_FLOAT) - { - id = isScalar ? NI_SSE_CompareScalarLessThanOrEqual : NI_SSE_CompareLessThanOrEqual; - } - else - { - assert(comp->compIsaSupportedDebugOnly(InstructionSet_SSE2)); - id = isScalar ? NI_SSE2_CompareScalarLessThanOrEqual : NI_SSE2_CompareLessThanOrEqual; - } -#elif defined(TARGET_ARM64) - if (genTypeSize(simdBaseType) == 8) - { - id = (simdSize == 8) ? NI_AdvSimd_Arm64_CompareLessThanOrEqualScalar - : NI_AdvSimd_Arm64_CompareLessThanOrEqual; - } - else - { - id = NI_AdvSimd_CompareLessThanOrEqual; - } -#endif // !TARGET_XARCH && !TARGET_ARM64 - break; - } - - case GT_LT: - { - assert(op2->TypeIs(simdType)); - -#if defined(TARGET_XARCH) - if (varTypeIsMask(type)) - { - id = NI_EVEX_CompareLessThanMask; - } - else if (varTypeIsIntegral(simdBaseType)) - { - if (varTypeIsSigned(simdBaseType)) - { - if (simdSize == 32) - { - assert(comp->compIsaSupportedDebugOnly(InstructionSet_AVX2)); - id = NI_AVX2_CompareLessThan; - } - else if (varTypeIsLong(simdBaseType)) - { - if (comp->compOpportunisticallyDependsOn(InstructionSet_SSE42)) - { - id = NI_SSE42_CompareLessThan; - } - } - else - { - assert(comp->compIsaSupportedDebugOnly(InstructionSet_SSE2)); - id = NI_SSE2_CompareLessThan; - } - } - else - { - // This should have been handled by the caller setting type = TYP_MASK - assert(!comp->canUseEvexEncodingDebugOnly()); - } - } - else if (simdSize == 32) - { - id = NI_AVX_CompareLessThan; - } - else if (simdBaseType == TYP_FLOAT) - { - id = isScalar ? NI_SSE_CompareScalarLessThan : NI_SSE_CompareLessThan; - } - else - { - assert(comp->compIsaSupportedDebugOnly(InstructionSet_SSE2)); - id = isScalar ? NI_SSE2_CompareScalarLessThan : NI_SSE2_CompareLessThan; - } -#elif defined(TARGET_ARM64) - if (genTypeSize(simdBaseType) == 8) - { - id = (simdSize == 8) ? NI_AdvSimd_Arm64_CompareLessThanScalar : NI_AdvSimd_Arm64_CompareLessThan; - } - else - { - id = NI_AdvSimd_CompareLessThan; - } -#endif // !TARGET_XARCH && !TARGET_ARM64 - break; - } - - case GT_NE: - { - assert(op2->TypeIs(simdType)); - -#if defined(TARGET_XARCH) - if (varTypeIsMask(type)) - { - id = NI_EVEX_CompareNotEqualMask; - } - else if (varTypeIsIntegral(simdBaseType)) - { - // This should have been handled by the caller setting type = TYP_MASK - assert(!comp->canUseEvexEncodingDebugOnly()); - } - else if (simdSize == 32) - { - id = NI_AVX_CompareNotEqual; - } - else if (simdBaseType == TYP_FLOAT) - { - id = isScalar ? NI_SSE_CompareScalarNotEqual : NI_SSE_CompareNotEqual; - } - else - { - assert(comp->compIsaSupportedDebugOnly(InstructionSet_SSE2)); - id = isScalar ? NI_SSE2_CompareScalarNotEqual : NI_SSE2_CompareNotEqual; - } -#endif // TARGET_XARCH - break; - } - - default: - { - unreached(); - } - } - - return id; -} - -//------------------------------------------------------------------------------ -// GetLookupTypeForCmpOp: Returns the lookup type for a SIMD comparison operation -// -// Arguments: -// oper - The comparison operation -// type - The expected IR type of the comparison -// simdBaseType - The base type on which oper is executed -// simdSize - The simd size on which oper is executed -// -// Returns: -// The lookup type for the given operation given the expected IR type, base type, and simd size -// -// Remarks: -// This API is namely meant to assist in handling cases where the underlying instruction return -// type doesn't match with the type IR wants us to be producing. For example, the consuming node -// may expect a TYP_SIMD16 but the underlying instruction may produce a TYP_MASK. -// -var_types GenTreeHWIntrinsic::GetLookupTypeForCmpOp( - Compiler* comp, genTreeOps oper, var_types type, var_types simdBaseType, unsigned simdSize) -{ - var_types simdType = comp->getSIMDTypeForSize(simdSize); - assert(varTypeIsMask(type) || (type == simdType)); - - assert(varTypeIsArithmetic(simdBaseType)); - assert(varTypeIsSIMD(simdType)); - - var_types lookupType = type; - -#if defined(TARGET_XARCH) - switch (oper) - { - case GT_EQ: - { - if (simdSize == 64) - { - lookupType = TYP_MASK; - } - break; - } - - case GT_GE: - case GT_LE: - case GT_NE: - { - if ((simdSize == 64) || (varTypeIsIntegral(simdBaseType) && comp->canUseEvexEncoding())) - { - lookupType = TYP_MASK; - } - break; - } - - case GT_GT: - case GT_LT: - { - if ((simdSize == 64) || (varTypeIsUnsigned(simdBaseType) && comp->canUseEvexEncoding())) - { - lookupType = TYP_MASK; - } - break; - } - - default: - { - unreached(); - } - } -#endif // TARGET_XARCH - - return lookupType; -} - -//--------------------------------------------------------------------------------------- -// GenTreeHWIntrinsic::ShouldConstantProp: -// Determines if a given operand should be constant propagated -// -// Arguments -// operand - The operand to check -// vecCon - The vector constant to check -// -// Return Value -// true if operand should be constant propagated, otherwise false -// -// Remarks -// This method takes the operand and vector constant given that assertion prop -// may be checking if the underlying constant for a lcl_var should be propagated -// -bool GenTreeHWIntrinsic::ShouldConstantProp(GenTree* operand, GenTreeVecCon* vecCon) -{ - assert(HWIntrinsicInfo::CanBenefitFromConstantProp(gtHWIntrinsicId)); - - var_types simdBaseType = GetSimdBaseType(); - - switch (gtHWIntrinsicId) - { -#if defined(TARGET_ARM64) - case NI_Vector64_op_Equality: - case NI_Vector64_op_Inequality: -#endif // TARGET_ARM64 - case NI_Vector128_op_Equality: - case NI_Vector128_op_Inequality: -#if defined(TARGET_XARCH) - case NI_Vector256_op_Equality: - case NI_Vector256_op_Inequality: - case NI_Vector512_op_Equality: - case NI_Vector512_op_Inequality: -#endif // TARGET_XARCH - { - // We can optimize when the constant is zero, but only - // for non floating-point since +0.0 == -0.0. - return vecCon->IsZero() && !varTypeIsFloating(simdBaseType); - } - -#if defined(TARGET_ARM64) - case NI_AdvSimd_CompareEqual: - case NI_AdvSimd_Arm64_CompareEqual: - case NI_AdvSimd_Arm64_CompareEqualScalar: - { - // We can optimize when the constant is zero due to a - // specialized encoding for the instruction - return vecCon->IsZero(); - } - - case NI_AdvSimd_CompareGreaterThan: - case NI_AdvSimd_CompareGreaterThanOrEqual: - case NI_AdvSimd_Arm64_CompareGreaterThan: - case NI_AdvSimd_Arm64_CompareGreaterThanOrEqual: - case NI_AdvSimd_Arm64_CompareGreaterThanScalar: - case NI_AdvSimd_Arm64_CompareGreaterThanOrEqualScalar: - { - // We can optimize when the constant is zero, but only - // for signed types, due to a specialized encoding for - // the instruction - return vecCon->IsZero() && !varTypeIsUnsigned(simdBaseType); - } -#endif // TARGET_ARM64 - -#if defined(TARGET_XARCH) - case NI_SSE41_Insert: - { - // We can optimize for float when the constant is zero - // due to a specialized encoding for the instruction - return (simdBaseType == TYP_FLOAT) && vecCon->IsZero(); - } - - case NI_EVEX_CompareEqualMask: - case NI_EVEX_CompareNotEqualMask: - { - // We can optimize when the constant is zero, but only - // for non floating-point since +0.0 == -0.0 - return vecCon->IsZero() && !varTypeIsFloating(simdBaseType); - } -#endif // TARGET_XARCH - - case NI_Vector128_Shuffle: -#if defined(TARGET_XARCH) - case NI_Vector256_Shuffle: - case NI_Vector512_Shuffle: -#elif defined(TARGET_ARM64) - case NI_Vector64_Shuffle: -#endif - { - // The shuffle indices need to be constant so we can preserve - // the node as a hwintrinsic instead of rewriting as a user call. - assert(GetOperandCount() == 2); - return IsUserCall() && (operand == Op(2)); - } - -#if defined(TARGET_XARCH) - case NI_SSE_Xor: - case NI_SSE2_Xor: - case NI_AVX_Xor: - case NI_AVX2_Xor: - case NI_AVX512F_Xor: - case NI_AVX512DQ_Xor: - case NI_AVX10v1_V512_Xor: - { - // We recognize this as GT_NOT which can enable other optimizations - assert(GetOperandCount() == 2); - return vecCon->IsVectorAllBitsSet(); - } -#endif // TARGET_XARCH - - default: - { - break; - } - } - - return false; -} -#endif // FEATURE_HW_INTRINSICS - -//--------------------------------------------------------------------------------------- -// gtNewMustThrowException: -// create a throw node (calling into JIT helper) that must be thrown. -// The result would be a comma node: COMMA(jithelperthrow(void), x) where x's type should be specified. -// -// Arguments -// helper - JIT helper ID -// type - return type of the node -// -// Return Value -// pointer to the throw node -// -GenTree* Compiler::gtNewMustThrowException(unsigned helper, var_types type, CORINFO_CLASS_HANDLE clsHnd) -{ - GenTreeCall* node = gtNewHelperCallNode(helper, TYP_VOID); - assert(node->IsNoReturn()); - if (type != TYP_VOID) - { - unsigned dummyTemp = lvaGrabTemp(true DEBUGARG("dummy temp of must thrown exception")); - if (type == TYP_STRUCT) - { - lvaSetStruct(dummyTemp, clsHnd, false); - type = lvaTable[dummyTemp].lvType; // struct type is normalized - } - else - { - lvaTable[dummyTemp].lvType = type; - } - GenTree* dummyNode = gtNewLclvNode(dummyTemp, type); - return gtNewOperNode(GT_COMMA, type, node, dummyNode); - } - return node; -} - -//--------------------------------------------------------------------------------------- -// InitializeStructReturnType: -// Initialize the Return Type Descriptor for a method that returns a struct type -// -// Arguments -// comp - Compiler Instance -// retClsHnd - VM handle to the struct type returned by the method -// -// Return Value -// None -// -void ReturnTypeDesc::InitializeStructReturnType(Compiler* comp, - CORINFO_CLASS_HANDLE retClsHnd, - CorInfoCallConvExtension callConv) -{ - assert(!m_inited); - - assert(retClsHnd != NO_CLASS_HANDLE); - unsigned structSize = comp->info.compCompHnd->getClassSize(retClsHnd); - - Compiler::structPassingKind howToReturnStruct; - var_types returnType = comp->getReturnTypeForStruct(retClsHnd, callConv, &howToReturnStruct, structSize); - - switch (howToReturnStruct) - { - case Compiler::SPK_EnclosingType: - case Compiler::SPK_PrimitiveType: - { - assert(returnType != TYP_UNKNOWN); - assert(returnType != TYP_STRUCT); - m_regType[0] = returnType; - -#if defined(TARGET_RISCV64) || defined(TARGET_LOONGARCH64) - const CORINFO_FPSTRUCT_LOWERING* lowering = comp->GetFpStructLowering(retClsHnd); - if (!lowering->byIntegerCallConv) - { - assert(lowering->numLoweredElements == 1); - m_fieldOffset[0] = lowering->offsets[0]; - } -#endif // defined(TARGET_RISCV64) || defined(TARGET_LOONGARCH64) - break; - } - - case Compiler::SPK_ByValueAsHfa: - { - assert(varTypeIsStruct(returnType)); - var_types hfaType = comp->GetHfaType(retClsHnd); - - // We should have an hfa struct type - assert(varTypeIsValidHfaType(hfaType)); - - // Note that the retail build issues a warning about a potential divsion by zero without this "max", - unsigned elemSize = max(1u, genTypeSize(hfaType)); - - // The size of this struct should be evenly divisible by elemSize - assert((structSize % elemSize) == 0); - - unsigned hfaCount = (structSize / elemSize); - for (unsigned i = 0; i < hfaCount; ++i) - { - m_regType[i] = hfaType; - } - - comp->compFloatingPointUsed = true; - break; - } - - case Compiler::SPK_ByValue: - { - assert(varTypeIsStruct(returnType)); - -#ifdef SWIFT_SUPPORT - if (callConv == CorInfoCallConvExtension::Swift) - { - InitializeSwiftReturnRegs(comp, retClsHnd); - break; - } -#endif - -#ifdef UNIX_AMD64_ABI - - SYSTEMV_AMD64_CORINFO_STRUCT_REG_PASSING_DESCRIPTOR structDesc; - comp->eeGetSystemVAmd64PassStructInRegisterDescriptor(retClsHnd, &structDesc); - - assert(structDesc.passedInRegisters); - for (int i = 0; i < structDesc.eightByteCount; i++) - { - assert(i < MAX_RET_REG_COUNT); - m_regType[i] = comp->GetEightByteType(structDesc, i); - } - -#elif defined(TARGET_ARM64) - - // a non-HFA struct returned using two registers - // - assert((structSize > TARGET_POINTER_SIZE) && (structSize <= (2 * TARGET_POINTER_SIZE))); - - BYTE gcPtrs[2] = {TYPE_GC_NONE, TYPE_GC_NONE}; - comp->info.compCompHnd->getClassGClayout(retClsHnd, &gcPtrs[0]); - for (unsigned i = 0; i < 2; ++i) - { - m_regType[i] = comp->getJitGCType(gcPtrs[i]); - } - -#elif defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64) - assert(structSize > sizeof(float)); - assert(structSize <= (2 * TARGET_POINTER_SIZE)); - BYTE gcPtrs[2] = {TYPE_GC_NONE, TYPE_GC_NONE}; - comp->info.compCompHnd->getClassGClayout(retClsHnd, &gcPtrs[0]); - const CORINFO_FPSTRUCT_LOWERING* lowering = comp->GetFpStructLowering(retClsHnd); - if (!lowering->byIntegerCallConv) - { - comp->compFloatingPointUsed = true; - assert(lowering->numLoweredElements == MAX_RET_REG_COUNT); - static_assert(MAX_RET_REG_COUNT == MAX_FPSTRUCT_LOWERED_ELEMENTS, ""); - for (unsigned i = 0; i < MAX_RET_REG_COUNT; ++i) - { - m_regType[i] = JITtype2varType(lowering->loweredElements[i]); - m_fieldOffset[i] = lowering->offsets[i]; - if (m_regType[i] == TYP_LONG && ((m_fieldOffset[i] % TARGET_POINTER_SIZE) == 0)) - { - unsigned slot = m_fieldOffset[i] / TARGET_POINTER_SIZE; - m_regType[i] = comp->getJitGCType(gcPtrs[slot]); - } - } - assert(varTypeIsFloating(m_regType[0]) || varTypeIsFloating(m_regType[1])); - } - else - { - for (unsigned i = 0; i < 2; ++i) - { - m_regType[i] = comp->getJitGCType(gcPtrs[i]); - } - m_fieldOffset[0] = 0; - m_fieldOffset[1] = TARGET_POINTER_SIZE; - } - -#elif defined(TARGET_X86) - - // an 8-byte struct returned using two registers - assert(structSize == 8); - - BYTE gcPtrs[2] = {TYPE_GC_NONE, TYPE_GC_NONE}; - comp->info.compCompHnd->getClassGClayout(retClsHnd, &gcPtrs[0]); - for (unsigned i = 0; i < 2; ++i) - { - m_regType[i] = comp->getJitGCType(gcPtrs[i]); - } - -#else // TARGET_XXX - - // This target needs support here! - // - NYI("Unsupported TARGET returning a TYP_STRUCT in InitializeStructReturnType"); - -#endif // UNIX_AMD64_ABI - break; // for case SPK_ByValue - } - - case Compiler::SPK_ByReference: - // We are returning using the return buffer argument, there are no return registers. - break; - - default: - unreached(); // By the contract of getReturnTypeForStruct we should never get here. - - } // end of switch (howToReturnStruct) - -#ifdef DEBUG - m_inited = true; -#endif -} - -#ifdef SWIFT_SUPPORT -//--------------------------------------------------------------------------------------- -// InitializeSwiftReturnRegs: -// Initialize the Return Type Descriptor for a method that returns with the -// Swift calling convention. -// -// Parameters: -// comp - Compiler instance -// clsHnd - Struct type being returned -// -void ReturnTypeDesc::InitializeSwiftReturnRegs(Compiler* comp, CORINFO_CLASS_HANDLE clsHnd) -{ - const CORINFO_SWIFT_LOWERING* lowering = comp->GetSwiftLowering(clsHnd); - assert(!lowering->byReference); - - static_assert_no_msg(MAX_SWIFT_LOWERED_ELEMENTS <= MAX_RET_REG_COUNT); - assert(lowering->numLoweredElements <= MAX_RET_REG_COUNT); - - for (size_t i = 0; i < lowering->numLoweredElements; i++) - { - m_regType[i] = JITtype2varType(lowering->loweredElements[i]); - } -} -#endif - -//--------------------------------------------------------------------------------------- -// InitializeLongReturnType: -// Initialize the Return Type Descriptor for a method that returns a TYP_LONG -// -void ReturnTypeDesc::InitializeLongReturnType() -{ - assert(!m_inited); -#if defined(TARGET_X86) || defined(TARGET_ARM) - // Setups up a ReturnTypeDesc for returning a long using two registers - // - assert(MAX_RET_REG_COUNT >= 2); - m_regType[0] = TYP_INT; - m_regType[1] = TYP_INT; - -#else // not (TARGET_X86 or TARGET_ARM) - - m_regType[0] = TYP_LONG; - -#endif // TARGET_X86 or TARGET_ARM - -#ifdef DEBUG - m_inited = true; -#endif -} - -//--------------------------------------------------------------------------------------- -// InitializeReturnType: Initialize the descriptor for a method that returns any type. -// -// Arguments: -// comp - The Compiler instance -// type - The return type as specififed by the signature -// retClsHnd - Handle for struct return types -// callConv - Method's calling convention -// -void ReturnTypeDesc::InitializeReturnType(Compiler* comp, - var_types type, - CORINFO_CLASS_HANDLE retClsHnd, - CorInfoCallConvExtension callConv) -{ - if (varTypeIsStruct(type)) - { - InitializeStructReturnType(comp, retClsHnd, callConv); - } - else if (type == TYP_LONG) - { - InitializeLongReturnType(); - } - else - { - if (type != TYP_VOID) - { - assert(varTypeIsEnregisterable(type)); - m_regType[0] = type; - } - - INDEBUG(m_inited = true); - } -} - -//------------------------------------------------------------------- -// GetABIReturnReg: Return i'th return register as per target ABI -// -// Arguments: -// idx - Index of the return register. -// The first return register has an index of 0 and so on. -// callConv - Associated calling convention -// -// Return Value: -// Returns i'th return register as per target ABI. -// -// Notes: -// x86 and ARM return long in multiple registers. -// ARM and ARM64 return HFA struct in multiple registers. -// -regNumber ReturnTypeDesc::GetABIReturnReg(unsigned idx, CorInfoCallConvExtension callConv) const -{ - unsigned count = GetReturnRegCount(); - assert(idx < count); - - regNumber resultReg = REG_NA; - -#ifdef SWIFT_SUPPORT - if (callConv == CorInfoCallConvExtension::Swift) - { - static const regNumber swiftIntReturnRegs[] = {REG_SWIFT_INTRET_ORDER}; - static const regNumber swiftFloatReturnRegs[] = {REG_SWIFT_FLOATRET_ORDER}; - assert((idx < ArrLen(swiftIntReturnRegs)) && (idx < ArrLen(swiftFloatReturnRegs))); - unsigned intRegIdx = 0; - unsigned floatRegIdx = 0; - for (unsigned i = 0; i < idx; i++) - { - if (varTypeUsesIntReg(GetReturnRegType(i))) - { - intRegIdx++; - } - else - { - floatRegIdx++; - } - } - - if (varTypeUsesIntReg(GetReturnRegType(idx))) - { - return swiftIntReturnRegs[intRegIdx]; - } - else - { - return swiftFloatReturnRegs[floatRegIdx]; - } - } -#endif - -#ifdef UNIX_AMD64_ABI - var_types regType0 = GetReturnRegType(0); - - if (idx == 0) - { - if (varTypeUsesIntReg(regType0)) - { - resultReg = REG_INTRET; - } - else - { - noway_assert(varTypeUsesFloatReg(regType0)); - resultReg = REG_FLOATRET; - } - } - else if (idx == 1) - { - var_types regType1 = GetReturnRegType(1); - - if (varTypeUsesIntReg(regType1)) - { - if (varTypeIsIntegralOrI(regType0)) - { - resultReg = REG_INTRET_1; - } - else - { - resultReg = REG_INTRET; - } - } - else - { - noway_assert(varTypeUsesFloatReg(regType1)); - - if (varTypeUsesFloatReg(regType0)) - { - resultReg = REG_FLOATRET_1; - } - else - { - resultReg = REG_FLOATRET; - } - } - } -#elif defined(WINDOWS_AMD64_ABI) - - assert(idx == 0); - - if (varTypeUsesIntReg(GetReturnRegType(0))) - { - resultReg = REG_INTRET; - } - else - { - assert(varTypeUsesFloatReg(GetReturnRegType(0))); - resultReg = REG_FLOATRET; - } - -#elif defined(TARGET_X86) - - if (idx == 0) - { - resultReg = REG_LNGRET_LO; - } - else if (idx == 1) - { - resultReg = REG_LNGRET_HI; - } - -#elif defined(TARGET_ARM) - - var_types regType = GetReturnRegType(idx); - if (varTypeIsIntegralOrI(regType)) - { - // Ints are returned in one return register. - // Longs are returned in two return registers. - if (idx == 0) - { - resultReg = REG_LNGRET_LO; - } - else if (idx == 1) - { - resultReg = REG_LNGRET_HI; - } - } - else - { - // Floats are returned in one return register (f0). - // Doubles are returned in one return register (d0). - // Structs are returned in four registers with HFAs. - assert(idx < MAX_RET_REG_COUNT); // Up to 4 return registers for HFA's - if (regType == TYP_DOUBLE) - { - resultReg = (regNumber)((unsigned)(REG_FLOATRET) + idx * 2); // d0, d1, d2 or d3 - } - else - { - resultReg = (regNumber)((unsigned)(REG_FLOATRET) + idx); // f0, f1, f2 or f3 - } - } - -#elif defined(TARGET_ARM64) - - var_types regType = GetReturnRegType(idx); - if (varTypeIsIntegralOrI(regType)) - { - noway_assert(idx < 2); // Up to 2 return registers for 16-byte structs - resultReg = (idx == 0) ? REG_INTRET : REG_INTRET_1; // X0 or X1 - } - else - { - noway_assert(idx < 4); // Up to 4 return registers for HFA's - resultReg = (regNumber)((unsigned)(REG_FLOATRET) + idx); // V0, V1, V2 or V3 - } - -#elif defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64) - var_types regType = GetReturnRegType(idx); - if (idx == 0) - { - resultReg = varTypeIsIntegralOrI(regType) ? REG_INTRET : REG_FLOATRET; // A0 or FA0 - } - else - { - noway_assert(idx == 1); // Up to 2 return registers for two-float-field structs - - // If the first return register is from the same register file, return the one next to it. - if (varTypeUsesIntReg(regType)) - { - resultReg = varTypeIsIntegralOrI(GetReturnRegType(0)) ? REG_INTRET_1 : REG_INTRET; // A0 or A1 - } - else - { - assert(varTypeUsesFloatReg(regType)); - resultReg = varTypeIsIntegralOrI(GetReturnRegType(0)) ? REG_FLOATRET : REG_FLOATRET_1; // FA0 or FA1 - } - } - -#endif // TARGET_XXX - - assert(resultReg != REG_NA); - return resultReg; -} - -//-------------------------------------------------------------------------------- -// GetABIReturnRegs: get the mask of return registers as per target arch ABI. -// -// Arguments: -// callConv - The calling convention -// -// Return Value: -// reg mask of return registers in which the return type is returned. -// -// Note: -// This routine can be used when the caller is not particular about the order -// of return registers and wants to know the set of return registers. -// -// static -regMaskTP ReturnTypeDesc::GetABIReturnRegs(CorInfoCallConvExtension callConv) const -{ - regMaskTP resultMask = RBM_NONE; - - unsigned count = GetReturnRegCount(); - for (unsigned i = 0; i < count; ++i) - { - resultMask |= genRegMask(GetABIReturnReg(i, callConv)); - } - - return resultMask; -} - -//------------------------------------------------------------------------ -// GetNum: Get the SSA number for a given field. -// -// Arguments: -// compiler - The Compiler instance -// index - The field index -// -// Return Value: -// The SSA number corresponding to the field at "index". -// -unsigned SsaNumInfo::GetNum(Compiler* compiler, unsigned index) const -{ - assert(IsComposite()); - if (HasCompactFormat()) - { - return (m_value >> (index * BITS_PER_SIMPLE_NUM)) & SIMPLE_NUM_MASK; - } - - // We expect this case to be very rare (outside stress). - return *GetOutlinedNumSlot(compiler, index); -} - -//------------------------------------------------------------------------ -// GetOutlinedNumSlot: Get a pointer the "outlined" SSA number for a field. -// -// Arguments: -// compiler - The Compiler instance -// index - The field index -// -// Return Value: -// Pointer to the SSA number corresponding to the field at "index". -// -unsigned* SsaNumInfo::GetOutlinedNumSlot(Compiler* compiler, unsigned index) const -{ - assert(IsComposite() && !HasCompactFormat()); - - // The "outlined" format for a composite number encodes a 30-bit-sized index. - // First, extract it: this will need "bit stitching" from the two parts. - unsigned outIndexLow = m_value & OUTLINED_INDEX_LOW_MASK; - unsigned outIndexHigh = (m_value & OUTLINED_INDEX_HIGH_MASK) >> 1; - unsigned outIndex = outIndexLow | outIndexHigh; - - return &compiler->m_outlinedCompositeSsaNums->GetRefNoExpand(outIndex + index); -} - -//------------------------------------------------------------------------ -// NumCanBeEncodedCompactly: Can the given field ref be encoded compactly? -// -// Arguments: -// ssaNum - The SSA number -// index - The field index -// -// Return Value: -// Whether the ref of the field at "index" can be encoded through the -// "compact" encoding scheme. -// -// Notes: -// Under stress, we randomly reduce the number of refs that can be -// encoded compactly, to stress the outlined encoding logic. -// -/* static */ bool SsaNumInfo::NumCanBeEncodedCompactly(unsigned index, unsigned ssaNum) -{ -#ifdef DEBUG - if (JitTls::GetCompiler()->compStressCompile(Compiler::STRESS_SSA_INFO, 20)) - { - return (ssaNum - 2) < index; - } -#endif // DEBUG - - assert(index < MAX_NumOfFieldsInPromotableStruct); - - return (ssaNum <= MAX_SIMPLE_NUM) && - ((index < SIMPLE_NUM_COUNT) || (SIMPLE_NUM_COUNT <= MAX_NumOfFieldsInPromotableStruct)); -} - -//------------------------------------------------------------------------ -// Composite: Form a composite SSA number, one capable of representing refs -// to more than one SSA local. -// -// Arguments: -// baseNum - The SSA number to base the new one on (composite/invalid) -// compiler - The Compiler instance -// parentLclNum - The promoted local representing a "whole" ref -// index - The field index -// ssaNum - The SSA number -// -// Return Value: -// A new, always composite, SSA number that represents all of the refs -// in "baseNum", with the field at "index" set to "ssaNum". -// -// Notes: -// It is assumed that the new number represents the same "whole" ref as -// the old one (the same parent local). If the SSA number needs to be -// reset fully, a new, RESERVED one should be created, and composed from -// with the appropriate parent reference. -// -/* static */ SsaNumInfo SsaNumInfo::Composite( - SsaNumInfo baseNum, Compiler* compiler, unsigned parentLclNum, unsigned index, unsigned ssaNum) -{ - assert(baseNum.IsInvalid() || baseNum.IsComposite()); - assert(compiler->lvaGetDesc(parentLclNum)->lvPromoted); - - if (NumCanBeEncodedCompactly(index, ssaNum) && (baseNum.IsInvalid() || baseNum.HasCompactFormat())) - { - unsigned ssaNumEncoded = ssaNum << (index * BITS_PER_SIMPLE_NUM); - if (baseNum.IsInvalid()) - { - return SsaNumInfo(COMPOSITE_ENCODING_BIT | ssaNumEncoded); - } - - return SsaNumInfo(ssaNumEncoded | (baseNum.m_value & ~(SIMPLE_NUM_MASK << (index * BITS_PER_SIMPLE_NUM)))); - } - - if (!baseNum.IsInvalid() && !baseNum.HasCompactFormat()) - { - *baseNum.GetOutlinedNumSlot(compiler, index) = ssaNum; - return baseNum; - } - - // This is the only path where we can encounter a null table. - if (compiler->m_outlinedCompositeSsaNums == nullptr) - { - CompAllocator alloc = compiler->getAllocator(CMK_SSA); - compiler->m_outlinedCompositeSsaNums = new (alloc) JitExpandArrayStack(alloc); - } - - // Allocate a new chunk for the field numbers. Once allocated, it cannot be expanded. - int count = compiler->lvaGetDesc(parentLclNum)->lvFieldCnt; - JitExpandArrayStack* table = compiler->m_outlinedCompositeSsaNums; - int outIdx = table->Size(); - unsigned* pLastSlot = &table->GetRef(outIdx + count - 1); // This will grow the table. - unsigned* pFirstSlot = pLastSlot - count + 1; - - // Copy over all of the already encoded numbers. - if (!baseNum.IsInvalid()) - { - for (int i = 0; i < SIMPLE_NUM_COUNT; i++) - { - pFirstSlot[i] = baseNum.GetNum(compiler, i); - } - } - - // Copy the one being set last to overwrite any previous values. - pFirstSlot[index] = ssaNum; - - // Split the index if it does not fit into a small encoding. - if ((outIdx & ~OUTLINED_INDEX_LOW_MASK) != 0) - { - int outIdxLow = outIdx & OUTLINED_INDEX_LOW_MASK; - int outIdxHigh = (outIdx << 1) & OUTLINED_INDEX_HIGH_MASK; - outIdx = outIdxLow | outIdxHigh; - } - - return SsaNumInfo(COMPOSITE_ENCODING_BIT | OUTLINED_ENCODING_BIT | outIdx); -} - -//------------------------------------------------------------------------ -// GetLclOffs: if `this` is a field or a field address it returns offset -// of the field inside the struct, for not a field it returns 0. -// -// Return Value: -// The offset value. -// -uint16_t GenTreeLclVarCommon::GetLclOffs() const -{ - if (OperIsLocalField()) - { - return AsLclFld()->GetLclOffs(); - } - else - { - return 0; - } -} - -//------------------------------------------------------------------------ -// GetLayout: get the struct layout for a local node of struct type. -// -// Arguments: -// compiler - the compiler instance -// -// Return Value: -// If "this" is a local field node, the layout stored in the node, -// otherwise the layout of local itself. -// -ClassLayout* GenTreeLclVarCommon::GetLayout(Compiler* compiler) const -{ - assert(varTypeIsStruct(TypeGet())); - - if (OperIs(GT_LCL_VAR, GT_STORE_LCL_VAR)) - { - return compiler->lvaGetDesc(GetLclNum())->GetLayout(); - } - - assert(OperIs(GT_LCL_FLD, GT_STORE_LCL_FLD)); - return AsLclFld()->GetLayout(); -} - -//------------------------------------------------------------------------ -// GenTreeLclVar::IsNeverNegative: Gets true if the lcl var is never negative; otherwise false. -// -// Arguments: -// comp - the compiler instance -// -// Return Value: -// true if the lcl var is never negative; otherwise false. -// -bool GenTreeLclVar::IsNeverNegative(Compiler* comp) const -{ - assert(OperIs(GT_LCL_VAR)); - return comp->lvaGetDesc(GetLclNum())->IsNeverNegative(); -} - -#if (defined(TARGET_XARCH) || defined(TARGET_ARM64)) && defined(FEATURE_HW_INTRINSICS) -//------------------------------------------------------------------------ -// GetResultOpNumForRmwIntrinsic: check if the result is written into one of the operands. -// In the case that none of the operand is overwritten, check if any of them is lastUse. -// -// Return Value: -// The operand number overwritten or lastUse. 0 is the default value, where the result is written into -// a destination that is not one of the source operands and there is no last use op. -// -unsigned GenTreeHWIntrinsic::GetResultOpNumForRmwIntrinsic(GenTree* use, GenTree* op1, GenTree* op2, GenTree* op3) -{ -#if defined(TARGET_XARCH) - assert(HWIntrinsicInfo::IsFmaIntrinsic(gtHWIntrinsicId) || HWIntrinsicInfo::IsPermuteVar2x(gtHWIntrinsicId) || - HWIntrinsicInfo::IsTernaryLogic(gtHWIntrinsicId)); -#elif defined(TARGET_ARM64) - assert(HWIntrinsicInfo::IsFmaIntrinsic(gtHWIntrinsicId)); -#endif - - if (use != nullptr && use->OperIs(GT_STORE_LCL_VAR)) - { - // For store_lcl_var, check if any op is overwritten - - GenTreeLclVarCommon* overwritten = use->AsLclVarCommon(); - unsigned overwrittenLclNum = overwritten->GetLclNum(); - - if (op1->IsLocal() && (op1->AsLclVarCommon()->GetLclNum() == overwrittenLclNum)) - { - return 1; - } - else if (op2->IsLocal() && (op2->AsLclVarCommon()->GetLclNum() == overwrittenLclNum)) - { - return 2; - } - else if (op3->IsLocal() && (op3->AsLclVarCommon()->GetLclNum() == overwrittenLclNum)) - { - return 3; - } - } - - // If no overwritten op, check if there is any last use op - // https://github.com/dotnet/runtime/issues/62215 - - if (op1->OperIs(GT_LCL_VAR) && op1->IsLastUse(0)) - { - return 1; - } - else if (op2->OperIs(GT_LCL_VAR) && op2->IsLastUse(0)) - { - return 2; - } - else if (op3->OperIs(GT_LCL_VAR) && op3->IsLastUse(0)) - { - return 3; - } - - return 0; -} -#endif // TARGET_XARCH && FEATURE_HW_INTRINSICS - -unsigned GenTreeLclFld::GetSize() const -{ - return TypeIs(TYP_STRUCT) ? GetLayout()->GetSize() : genTypeSize(TypeGet()); -} - -#ifdef TARGET_ARM -//------------------------------------------------------------------------ -// IsOffsetMisaligned: check if the field needs a special handling on arm. -// -// Return Value: -// true if it is a float field with a misaligned offset, false otherwise. -// -bool GenTreeLclFld::IsOffsetMisaligned() const -{ - if (varTypeIsFloating(gtType)) - { - return ((m_lclOffs % emitTypeSize(TYP_FLOAT)) != 0); - } - return false; -} -#endif // TARGET_ARM - -bool GenTree::IsInvariant() const -{ - return OperIsConst() || OperIs(GT_LCL_ADDR) || OperIs(GT_FTN_ADDR); -} - -//------------------------------------------------------------------- -// IsVectorPerElementMask: returns true if this node is a vector constant per-element mask -// (every element has either all bits set or none of them). -// -// Arguments: -// simdBaseType - the base type of the constant being checked. -// simdSize - the size of the SIMD type of the intrinsic. -// -// Returns: -// True if this node is a vector constant per-element mask. -// -bool GenTree::IsVectorPerElementMask(var_types simdBaseType, unsigned simdSize) const -{ -#ifdef FEATURE_SIMD - if (IsCnsVec()) - { - const GenTreeVecCon* vecCon = AsVecCon(); - - int elementCount = vecCon->ElementCount(simdSize, simdBaseType); - - switch (simdBaseType) - { - case TYP_BYTE: - case TYP_UBYTE: - return ElementsAreAllBitsSetOrZero(&vecCon->gtSimdVal.u8[0], elementCount); - case TYP_SHORT: - case TYP_USHORT: - return ElementsAreAllBitsSetOrZero(&vecCon->gtSimdVal.u16[0], elementCount); - case TYP_INT: - case TYP_UINT: - case TYP_FLOAT: - return ElementsAreAllBitsSetOrZero(&vecCon->gtSimdVal.u32[0], elementCount); - case TYP_LONG: - case TYP_ULONG: - case TYP_DOUBLE: - return ElementsAreAllBitsSetOrZero(&vecCon->gtSimdVal.u64[0], elementCount); - default: - unreached(); - } - } - else if (OperIsHWIntrinsic()) - { - const GenTreeHWIntrinsic* intrinsic = AsHWIntrinsic(); - const NamedIntrinsic intrinsicId = intrinsic->GetHWIntrinsicId(); - - if (HWIntrinsicInfo::ReturnsPerElementMask(intrinsicId)) - { - // We directly return a per-element mask - return true; - } - - bool isScalar = false; - genTreeOps oper = intrinsic->GetOperForHWIntrinsicId(&isScalar); - - switch (oper) - { - case GT_AND: - case GT_AND_NOT: - case GT_OR: - case GT_XOR: - { - // We are a binary bitwise operation where both inputs are per-element masks - return intrinsic->Op(1)->IsVectorPerElementMask(simdBaseType, simdSize) && - intrinsic->Op(2)->IsVectorPerElementMask(simdBaseType, simdSize); - } - - case GT_NOT: - { - // We are an unary bitwise operation where the input is a per-element mask - return intrinsic->Op(1)->IsVectorPerElementMask(simdBaseType, simdSize); - } - - default: - { - assert(!GenTreeHWIntrinsic::OperIsBitwiseHWIntrinsic(oper)); - break; - } - } - - return false; - } - else if (IsCnsMsk()) - { - return true; - } -#endif // FEATURE_SIMD - - return false; -} - -//------------------------------------------------------------------------ -// IsNeverNegative: returns true if the given tree is known to be never -// negative, i. e. the upper bit will always be zero. -// Only valid for integral types. -// -// Arguments: -// comp - Compiler object, needed for IntegralRange::ForNode -// -// Return Value: -// true if the given tree is known to be never negative -// -bool GenTree::IsNeverNegative(Compiler* comp) const -{ - assert(varTypeIsIntegral(this)); - - if (IsIntegralConst()) - { - return AsIntConCommon()->IntegralValue() >= 0; - } - - if (OperIs(GT_LCL_VAR)) - { - if (AsLclVar()->IsNeverNegative(comp)) - { - // This is an early exit, it doesn't cover all cases - return true; - } - } - - if (IntegralRange::ForNode(const_cast(this), comp).IsNonNegative()) - { - return true; - } - - if ((comp->vnStore != nullptr) && comp->vnStore->IsVNNeverNegative(gtVNPair.GetConservative())) - { - return true; - } - - return false; -} - -//------------------------------------------------------------------------ -// IsNeverNegativeOne: returns true if the given tree is known to never be -// be negative one. Only valid for integral types. -// -// Arguments: -// comp - Compiler object, needed for IsNeverNegative -// -// Return Value: -// true if the given tree is known to never be negative one -// -bool GenTree::IsNeverNegativeOne(Compiler* comp) const -{ - assert(varTypeIsIntegral(this)); - - if (this->IsNeverNegative(comp)) - return true; - - if (this->IsIntegralConst()) - { - return !this->IsIntegralConst(-1); - } - - return false; -} - -//------------------------------------------------------------------------ -// IsNeverZero: returns true if the given tree is known to never be zero. -// Only valid for integral types. -// -// Return Value: -// true if the given tree is known to never be zero -// -bool GenTree::IsNeverZero() const -{ - assert(varTypeIsIntegral(this)); - - if (this->IsIntegralConst()) - { - return !this->IsIntegralConst(0); - } - - return false; -} - -//------------------------------------------------------------------------ -// CanDivOrModPossiblyOverflow: returns true if the given tree is known -// to possibly overflow on a division. -// Only valid for integral types. -// Only valid for signed-div/signed-mod. -// -// Arguments: -// comp - Compiler object, needed for IsNeverNegativeOne -// -// Return Value: -// true if the given tree is known to possibly overflow on a division -// -bool GenTree::CanDivOrModPossiblyOverflow(Compiler* comp) const -{ - assert(this->OperIs(GT_DIV, GT_MOD)); - assert(varTypeIsIntegral(this)); - - if (this->gtFlags & GTF_DIV_MOD_NO_OVERFLOW) - return false; - - GenTree* op1 = this->gtGetOp1()->gtSkipReloadOrCopy(); - GenTree* op2 = this->gtGetOp2()->gtSkipReloadOrCopy(); - - // If the divisor is known to never be '-1', we cannot overflow. - if (op2->IsNeverNegativeOne(comp)) - return false; - - // If the dividend is a constant with a minimum value with respect to the division's type, then we might overflow - // as we do not know if the divisor will be '-1' or not at this point. - if (op1->IsIntegralConst()) - { - if (this->TypeIs(TYP_INT) && op1->IsIntegralConst(INT32_MIN)) - { - return true; - } - else if (this->TypeIs(TYP_LONG) && (op1->AsIntConCommon()->IntegralValue() == INT64_MIN)) - { - return true; - } - - // Dividend is not a minimum value; therefore we cannot overflow. - return false; - } - - // Not enough known information; therefore we might overflow. - return true; -} - -#if defined(FEATURE_HW_INTRINSICS) -GenTree* Compiler::gtFoldExprHWIntrinsic(GenTreeHWIntrinsic* tree) -{ - if (!opts.Tier0OptimizationEnabled()) - { - return tree; - } - - NamedIntrinsic ni = tree->GetHWIntrinsicId(); - var_types retType = tree->TypeGet(); - var_types simdBaseType = tree->GetSimdBaseType(); - CorInfoType simdBaseJitType = tree->GetSimdBaseJitType(); - unsigned int simdSize = tree->GetSimdSize(); - - simd_t simdVal = {}; - - if (GenTreeVecCon::IsHWIntrinsicCreateConstant(tree, simdVal)) - { - GenTreeVecCon* vecCon = gtNewVconNode(retType); - - for (GenTree* arg : tree->Operands()) - { - DEBUG_DESTROY_NODE(arg); - } - - vecCon->gtSimdVal = simdVal; - vecCon->SetMorphed(this); - fgUpdateConstTreeValueNumber(vecCon); - return vecCon; - } - - GenTree* op1 = nullptr; - GenTree* op2 = nullptr; - GenTree* op3 = nullptr; - size_t opCount = tree->GetOperandCount(); - - switch (opCount) - { - case 3: - { - op3 = tree->Op(3); - FALLTHROUGH; - } - - case 2: - { - op2 = tree->Op(2); - FALLTHROUGH; - } - - case 1: - { - op1 = tree->Op(1); - break; - } - - default: - { - return tree; - } - } - -#if defined(FEATURE_MASKED_HW_INTRINSICS) - if (tree->OperIsConvertMaskToVector()) - { - GenTree* op = op1; - - if (op->OperIsHWIntrinsic()) - { - unsigned simdBaseTypeSize = genTypeSize(simdBaseType); - GenTreeHWIntrinsic* cvtOp = op->AsHWIntrinsic(); - - if (cvtOp->OperIsConvertVectorToMask()) - { - if ((genTypeSize(cvtOp->GetSimdBaseType()) == simdBaseTypeSize)) - { - // We need the operand to be the same kind of mask; otherwise - // the bitwise operation can differ in how it performs - -#if defined(TARGET_XARCH) - GenTree* vectorNode = cvtOp->Op(1); -#elif defined(TARGET_ARM64) - GenTree* vectorNode = cvtOp->Op(2); -#else -#error Unsupported platform -#endif // !TARGET_XARCH && !TARGET_ARM64 - - DEBUG_DESTROY_NODE(op, tree); - vectorNode->SetMorphed(this); - return vectorNode; - } - } - } - } - - if (tree->OperIsConvertVectorToMask()) - { - GenTree* op = op1; - bool tryHandle = false; - -#if defined(TARGET_XARCH) - tryHandle = op->OperIsHWIntrinsic(); -#elif defined(TARGET_ARM64) - if (op->OperIsHWIntrinsic(NI_Sve_CreateTrueMaskAll)) - { - op = op2; - tryHandle = op->OperIsHWIntrinsic(); - } -#endif // TARGET_ARM64 - - if (tryHandle) - { - unsigned simdBaseTypeSize = genTypeSize(simdBaseType); - GenTreeHWIntrinsic* cvtOp = op->AsHWIntrinsic(); - - if (cvtOp->OperIsConvertMaskToVector()) - { - if ((genTypeSize(cvtOp->GetSimdBaseType()) == simdBaseTypeSize)) - { - // We need the operand to be the same kind of mask; otherwise - // the bitwise operation can differ in how it performs - - GenTree* maskNode = cvtOp->Op(1); - -#if defined(TARGET_ARM64) - DEBUG_DESTROY_NODE(op1); -#endif // TARGET_ARM64 - - DEBUG_DESTROY_NODE(op, tree); - return maskNode; - } - } - } - } -#else - assert(!tree->OperIsConvertMaskToVector()); - assert(!tree->OperIsConvertVectorToMask()); -#endif // FEATURE_MASKED_HW_INTRINSICS - - bool isScalar = false; - genTreeOps oper = tree->GetOperForHWIntrinsicId(&isScalar); - - // We shouldn't find AND_NOT nodes since it should only be produced in lowering - assert(oper != GT_AND_NOT); - - GenTree* cnsNode = nullptr; - GenTree* otherNode = nullptr; - - if (op1->OperIsConst()) - { - cnsNode = op1; - otherNode = op2; - } - else if ((op2 != nullptr) && op2->OperIsConst()) - { - cnsNode = op2; - otherNode = op1; - } - else - { - // No constants, so nothing to fold - return tree; - } - - GenTree* resultNode = tree; - - if (otherNode == nullptr) - { - assert(op2 == nullptr); - assert(op3 == nullptr); - - if (oper != GT_NONE) - { - if (varTypeIsMask(retType)) - { - cnsNode->AsMskCon()->EvaluateUnaryInPlace(oper, isScalar, simdBaseType, simdSize); - } - else - { - cnsNode->AsVecCon()->EvaluateUnaryInPlace(oper, isScalar, simdBaseType); - } - resultNode = cnsNode; - } -#if defined(FEATURE_MASKED_HW_INTRINSICS) - else if (tree->OperIsConvertMaskToVector()) - { - GenTreeMskCon* mskCon = cnsNode->AsMskCon(); - - simd_t simdVal; - EvaluateSimdCvtMaskToVector(simdBaseType, &simdVal, mskCon->gtSimdMaskVal); - - resultNode = gtNewVconNode(retType, &simdVal); - } - else if (tree->OperIsConvertVectorToMask()) - { - GenTreeVecCon* vecCon = cnsNode->AsVecCon(); - GenTreeMskCon* mskCon = gtNewMskConNode(retType); - - switch (vecCon->TypeGet()) - { - case TYP_SIMD8: - { - EvaluateSimdCvtVectorToMask(simdBaseType, &mskCon->gtSimdMaskVal, vecCon->gtSimd8Val); - break; - } - - case TYP_SIMD12: - { - EvaluateSimdCvtVectorToMask(simdBaseType, &mskCon->gtSimdMaskVal, vecCon->gtSimd12Val); - break; - } - - case TYP_SIMD16: - { - EvaluateSimdCvtVectorToMask(simdBaseType, &mskCon->gtSimdMaskVal, vecCon->gtSimd16Val); - break; - } - -#if defined(TARGET_XARCH) - case TYP_SIMD32: - { - EvaluateSimdCvtVectorToMask(simdBaseType, &mskCon->gtSimdMaskVal, vecCon->gtSimd32Val); - break; - } - - case TYP_SIMD64: - { - EvaluateSimdCvtVectorToMask(simdBaseType, &mskCon->gtSimdMaskVal, vecCon->gtSimd64Val); - break; - } -#endif // TARGET_XARCH - - default: - { - unreached(); - } - } - - resultNode = mskCon; - } -#endif // FEATURE_MASKED_HW_INTRINSICS - else - { - switch (ni) - { -#ifdef TARGET_ARM64 - case NI_ArmBase_LeadingZeroCount: -#else - case NI_LZCNT_LeadingZeroCount: -#endif - { - assert(!varTypeIsSmall(retType) && !varTypeIsLong(retType)); - - int32_t value = static_cast(cnsNode->AsIntConCommon()->IconValue()); - uint32_t result = BitOperations::LeadingZeroCount(static_cast(value)); - - cnsNode->AsIntConCommon()->SetIconValue(static_cast(result)); - resultNode = cnsNode; - break; - } - -#ifdef TARGET_ARM64 - case NI_ArmBase_Arm64_LeadingZeroCount: - { - assert(varTypeIsInt(retType)); - - int64_t value = cnsNode->AsIntConCommon()->IntegralValue(); - uint32_t result = BitOperations::LeadingZeroCount(static_cast(value)); - - cnsNode->AsIntConCommon()->SetIconValue(static_cast(result)); - cnsNode->gtType = retType; - - resultNode = cnsNode; - break; - } -#else - case NI_LZCNT_X64_LeadingZeroCount: - { - assert(varTypeIsLong(retType)); - - int64_t value = cnsNode->AsIntConCommon()->IntegralValue(); - uint32_t result = BitOperations::LeadingZeroCount(static_cast(value)); - - cnsNode->AsIntConCommon()->SetIntegralValue(static_cast(result)); - resultNode = cnsNode; - break; - } -#endif - - case NI_Vector128_AsVector3: - case NI_Vector128_AsVector128Unsafe: -#ifdef TARGET_ARM64 - case NI_Vector64_ToVector128Unsafe: - case NI_Vector128_GetLower: -#else - case NI_Vector128_AsVector2: - case NI_Vector128_ToVector256Unsafe: - case NI_Vector256_GetLower: - case NI_Vector256_ToVector512Unsafe: - case NI_Vector512_GetLower: - case NI_Vector512_GetLower128: -#endif - { - // These are all going to a smaller type taking the lowest bits - // or are unsafely going to a larger type, so we just need to retype - // the constant and we're good to go. - - cnsNode->gtType = retType; - resultNode = cnsNode; - break; - } - -#ifdef TARGET_ARM64 - case NI_Vector64_ToVector128: - { - assert(retType == TYP_SIMD16); - assert(cnsNode->gtType == TYP_SIMD8); - cnsNode->AsVecCon()->gtSimd16Val.v64[1] = {}; - - cnsNode->gtType = retType; - resultNode = cnsNode; - break; - } -#else - case NI_Vector128_ToVector256: - { - assert(retType == TYP_SIMD32); - assert(cnsNode->gtType == TYP_SIMD16); - cnsNode->AsVecCon()->gtSimd32Val.v128[1] = {}; - - cnsNode->gtType = retType; - resultNode = cnsNode; - break; - } - - case NI_Vector128_ToVector512: - { - assert(retType == TYP_SIMD64); - assert(cnsNode->gtType == TYP_SIMD16); - cnsNode->AsVecCon()->gtSimd64Val.v128[1] = {}; - cnsNode->AsVecCon()->gtSimd64Val.v256[1] = {}; - - cnsNode->gtType = retType; - resultNode = cnsNode; - break; - } - - case NI_Vector256_ToVector512: - { - assert(retType == TYP_SIMD64); - assert(cnsNode->gtType == TYP_SIMD32); - cnsNode->AsVecCon()->gtSimd64Val.v256[1] = {}; - - cnsNode->gtType = retType; - resultNode = cnsNode; - break; - } -#endif - -#ifdef TARGET_ARM64 - case NI_Vector128_GetUpper: - { - assert(retType == TYP_SIMD8); - assert(cnsNode->gtType == TYP_SIMD16); - cnsNode->AsVecCon()->gtSimd8Val = cnsNode->AsVecCon()->gtSimd16Val.v64[1]; - - cnsNode->gtType = retType; - resultNode = cnsNode; - break; - } -#else - case NI_Vector256_GetUpper: - { - assert(retType == TYP_SIMD16); - assert(cnsNode->gtType == TYP_SIMD32); - cnsNode->AsVecCon()->gtSimd16Val = cnsNode->AsVecCon()->gtSimd32Val.v128[1]; - - cnsNode->gtType = retType; - resultNode = cnsNode; - break; - } - - case NI_Vector512_GetUpper: - { - assert(retType == TYP_SIMD32); - assert(cnsNode->gtType == TYP_SIMD64); - cnsNode->AsVecCon()->gtSimd32Val = cnsNode->AsVecCon()->gtSimd64Val.v256[1]; - - cnsNode->gtType = retType; - resultNode = cnsNode; - break; - } -#endif - - case NI_Vector128_ToScalar: -#ifdef TARGET_ARM64 - case NI_Vector64_ToScalar: -#else - case NI_Vector256_ToScalar: - case NI_Vector512_ToScalar: -#endif - { - var_types simdType = getSIMDTypeForSize(simdSize); - - if (varTypeIsFloating(retType)) - { - double result = cnsNode->AsVecCon()->ToScalarFloating(simdBaseType); - - resultNode = gtNewDconNode(result, retType); - } - else - { - assert(varTypeIsIntegral(retType)); - int64_t result = cnsNode->AsVecCon()->ToScalarIntegral(simdBaseType); - - if (varTypeIsLong(retType)) - { - resultNode = gtNewLconNode(result); - } - else - { - resultNode = gtNewIconNode(static_cast(result), retType); - } - } - break; - } - - default: - { - break; - } - } - } - } - else if (otherNode->OperIsConst()) - { - if (oper != GT_NONE) - { - assert(op3 == nullptr); - - if (varTypeIsMask(retType)) - { - if (varTypeIsMask(cnsNode)) - { - cnsNode->AsMskCon()->EvaluateBinaryInPlace(oper, isScalar, simdBaseType, simdSize, - otherNode->AsMskCon()); - } - else - { - cnsNode->AsVecCon()->EvaluateBinaryInPlace(oper, isScalar, simdBaseType, otherNode->AsVecCon()); - } - } - else - { -#if defined(TARGET_XARCH) - if ((oper == GT_LSH) || (oper == GT_RSH) || (oper == GT_RSZ)) - { - if (otherNode->TypeIs(TYP_SIMD16)) - { - if (!HWIntrinsicInfo::IsVariableShift(ni)) - { - // The xarch shift instructions support taking the shift amount as - // a simd16, in which case they take the shift amount from the lower - // 64-bits. - - int64_t shiftAmount = otherNode->AsVecCon()->GetElementIntegral(TYP_LONG, 0); - - if (static_cast(shiftAmount) >= - (static_cast(genTypeSize(simdBaseType)) * BITS_PER_BYTE)) - { - // Set to -1 to indicate an explicit overshift - shiftAmount = -1; - } - - // Ensure we broadcast to the right vector size - otherNode->gtType = retType; - - otherNode->AsVecCon()->EvaluateBroadcastInPlace(simdBaseType, shiftAmount); - } - } - } -#endif // TARGET_XARCH - - if (otherNode->IsIntegralConst()) - { - int64_t scalar = otherNode->AsIntConCommon()->IntegralValue(); - - otherNode = gtNewVconNode(retType); - otherNode->AsVecCon()->EvaluateBroadcastInPlace(simdBaseType, scalar); - } - - cnsNode->AsVecCon()->EvaluateBinaryInPlace(oper, isScalar, simdBaseType, otherNode->AsVecCon()); - } - resultNode = cnsNode; - } - else - { - switch (ni) - { - case NI_Vector128_GetElement: -#ifdef TARGET_ARM64 - case NI_Vector64_GetElement: -#else - case NI_Vector256_GetElement: - case NI_Vector512_GetElement: -#endif - { - assert(op3 == nullptr); - uint32_t index = static_cast(otherNode->AsIntConCommon()->IconValue()); - - if (index >= GenTreeVecCon::ElementCount(simdSize, simdBaseType)) - { - // Nothing to fold for out of range indexes - break; - } - - var_types simdType = getSIMDTypeForSize(simdSize); - - if (varTypeIsFloating(retType)) - { - double result = cnsNode->AsVecCon()->GetElementFloating(simdBaseType, index); - - resultNode = gtNewDconNode(result, retType); - } - else - { - assert(varTypeIsIntegral(retType)); - int64_t result = cnsNode->AsVecCon()->GetElementIntegral(simdBaseType, index); - - if (varTypeIsLong(retType)) - { - resultNode = gtNewLconNode(result); - } - else - { - resultNode = gtNewIconNode(static_cast(result), retType); - } - } - break; - } - -#ifdef TARGET_ARM64 - case NI_AdvSimd_MultiplyByScalar: - case NI_AdvSimd_Arm64_MultiplyByScalar: - { - assert(op3 == nullptr); - - // MultiplyByScalar takes a vector as the second operand but only utilizes element 0 - // We need to extract it and then functionally broadcast it up for the evaluation to - // work as expected. Ensuring we broadcast up to the target vector size. - - otherNode->gtType = retType; - - if (varTypeIsFloating(simdBaseType)) - { - double scalar = otherNode->AsVecCon()->ToScalarFloating(simdBaseType); - otherNode->AsVecCon()->EvaluateBroadcastInPlace(simdBaseType, scalar); - } - else - { - assert(varTypeIsIntegral(simdBaseType)); - int64_t scalar = otherNode->AsVecCon()->ToScalarIntegral(simdBaseType); - otherNode->AsVecCon()->EvaluateBroadcastInPlace(simdBaseType, scalar); - } - - cnsNode->AsVecCon()->EvaluateBinaryInPlace(GT_MUL, isScalar, simdBaseType, otherNode->AsVecCon()); - resultNode = cnsNode; - break; - } -#endif - - case NI_Vector128_WithElement: -#ifdef TARGET_ARM64 - case NI_Vector64_WithElement: -#else - case NI_Vector256_WithElement: - case NI_Vector512_WithElement: -#endif - { - if (!op3->OperIsConst()) - { - break; - } - - uint32_t index = static_cast(op2->AsIntConCommon()->IconValue()); - - if (index >= GenTreeVecCon::ElementCount(simdSize, simdBaseType)) - { - // Nothing to fold for out of range indexes - break; - } - - var_types simdType = getSIMDTypeForSize(simdSize); - - if (varTypeIsFloating(simdBaseType)) - { - double value = op3->AsDblCon()->DconValue(); - cnsNode->AsVecCon()->SetElementFloating(simdBaseType, index, value); - resultNode = cnsNode; - } - else - { - assert(varTypeIsIntegral(simdBaseType)); - int64_t value = op3->AsIntConCommon()->IntegralValue(); - cnsNode->AsVecCon()->SetElementIntegral(simdBaseType, index, value); - resultNode = cnsNode; - } - break; - } - -#ifdef TARGET_ARM64 - case NI_Vector128_WithLower: - { - assert(retType == TYP_SIMD16); - assert(cnsNode->gtType == TYP_SIMD16); - assert(otherNode->gtType == TYP_SIMD8); - cnsNode->AsVecCon()->gtSimd16Val.v64[0] = otherNode->AsVecCon()->gtSimd8Val; - - resultNode = cnsNode; - break; - } -#else - case NI_Vector256_WithLower: - { - assert(retType == TYP_SIMD32); - assert(cnsNode->gtType == TYP_SIMD32); - assert(otherNode->gtType == TYP_SIMD16); - cnsNode->AsVecCon()->gtSimd32Val.v128[0] = otherNode->AsVecCon()->gtSimd16Val; - - resultNode = cnsNode; - break; - } - - case NI_Vector512_WithLower: - { - assert(retType == TYP_SIMD64); - assert(cnsNode->gtType == TYP_SIMD64); - assert(otherNode->gtType == TYP_SIMD32); - cnsNode->AsVecCon()->gtSimd64Val.v256[0] = otherNode->AsVecCon()->gtSimd32Val; - - resultNode = cnsNode; - break; - } -#endif - -#ifdef TARGET_ARM64 - case NI_Vector128_WithUpper: - { - assert(retType == TYP_SIMD16); - assert(cnsNode->gtType == TYP_SIMD16); - assert(otherNode->gtType == TYP_SIMD8); - cnsNode->AsVecCon()->gtSimd16Val.v64[1] = otherNode->AsVecCon()->gtSimd8Val; - - resultNode = cnsNode; - break; - } -#else - case NI_Vector256_WithUpper: - { - assert(retType == TYP_SIMD32); - assert(cnsNode->gtType == TYP_SIMD32); - assert(otherNode->gtType == TYP_SIMD16); - cnsNode->AsVecCon()->gtSimd32Val.v128[1] = otherNode->AsVecCon()->gtSimd16Val; - - resultNode = cnsNode; - break; - } - - case NI_Vector512_WithUpper: - { - assert(retType == TYP_SIMD64); - assert(cnsNode->gtType == TYP_SIMD64); - assert(otherNode->gtType == TYP_SIMD32); - cnsNode->AsVecCon()->gtSimd64Val.v256[1] = otherNode->AsVecCon()->gtSimd32Val; - - resultNode = cnsNode; - break; - } -#endif - - case NI_Vector128_op_Equality: -#if defined(TARGET_ARM64) - case NI_Vector64_op_Equality: -#elif defined(TARGET_XARCH) - case NI_Vector256_op_Equality: - case NI_Vector512_op_Equality: -#endif // !TARGET_ARM64 && !TARGET_XARCH - { - cnsNode->AsVecCon()->EvaluateBinaryInPlace(GT_EQ, isScalar, simdBaseType, otherNode->AsVecCon()); - resultNode = gtNewIconNode(cnsNode->AsVecCon()->IsAllBitsSet() ? 1 : 0, retType); - break; - } - - case NI_Vector128_op_Inequality: -#if defined(TARGET_ARM64) - case NI_Vector64_op_Inequality: -#elif defined(TARGET_XARCH) - case NI_Vector256_op_Inequality: - case NI_Vector512_op_Inequality: -#endif // !TARGET_ARM64 && !TARGET_XARCH - { - cnsNode->AsVecCon()->EvaluateBinaryInPlace(GT_NE, isScalar, simdBaseType, otherNode->AsVecCon()); - resultNode = gtNewIconNode(cnsNode->AsVecCon()->IsZero() ? 0 : 1, retType); - break; - } - - default: - { - break; - } - } - } - } - else if (op3 == nullptr) - { - if (isScalar) - { - // We don't support folding for scalars when only one input is constant - // because it means one value is computed and the remaining values are - // either zeroed or preserved based on the underlying target architecture - oper = GT_NONE; - } - - switch (oper) - { - case GT_ADD: - { - if (varTypeIsFloating(simdBaseType)) - { - // Handle `x + NaN == NaN` and `NaN + x == NaN` - // This is safe for all floats since we do not fault for sNaN - - if (cnsNode->IsVectorNaN(simdBaseType)) - { - resultNode = gtWrapWithSideEffects(cnsNode, otherNode, GTF_ALL_EFFECT); - break; - } - - // Handle `x + -0 == x` and `-0 + x == x` - - if (cnsNode->IsVectorNegativeZero(simdBaseType)) - { - resultNode = otherNode; - break; - } - - // We cannot handle `x + 0 == x` or `0 + x == x` since `-0 + 0 == 0` - break; - } - - // Handle `x + 0 == x` and `0 + x == x` - if (cnsNode->IsVectorZero()) - { - resultNode = otherNode; - } - break; - } - - case GT_AND: - { - // Handle `x & 0 == 0` and `0 & x == 0` - if (cnsNode->IsVectorZero()) - { - resultNode = gtWrapWithSideEffects(cnsNode, otherNode, GTF_ALL_EFFECT); - break; - } - - // Handle `x & AllBitsSet == x` and `AllBitsSet & x == x` - if (cnsNode->IsVectorAllBitsSet()) - { - resultNode = otherNode; - } - break; - } - - case GT_DIV: - { - if (varTypeIsFloating(simdBaseType)) - { - // Handle `x / NaN == NaN` and `NaN / x == NaN` - // This is safe for all floats since we do not fault for sNaN - - if (cnsNode->IsVectorNaN(simdBaseType)) - { - resultNode = gtWrapWithSideEffects(cnsNode, otherNode, GTF_ALL_EFFECT); - break; - } - } - - // Handle `x / 1 == x`. - // This is safe for all floats since we do not fault for sNaN - - if (cnsNode != op2) - { - break; - } - - if (!cnsNode->IsVectorBroadcast(simdBaseType)) - { - break; - } - - if (cnsNode->AsVecCon()->IsScalarOne(simdBaseType)) - { - resultNode = otherNode; - } - break; - } - - case GT_EQ: - { - if (varTypeIsFloating(simdBaseType)) - { - // Handle `(x == NaN) == false` and `(NaN == x) == false` for floating-point types - if (cnsNode->IsVectorNaN(simdBaseType)) - { - int64_t zero = 0; - cnsNode->AsVecCon()->EvaluateBroadcastInPlace(TYP_LONG, zero); - resultNode = gtWrapWithSideEffects(cnsNode, otherNode, GTF_ALL_EFFECT); - break; - } - } - break; - } - - case GT_GT: - { - if (varTypeIsUnsigned(simdBaseType)) - { - // Handle `(0 > x) == false` for unsigned types. - if ((cnsNode == op1) && cnsNode->IsVectorZero()) - { - int64_t zero = 0; - cnsNode->AsVecCon()->EvaluateBroadcastInPlace(TYP_LONG, zero); - resultNode = gtWrapWithSideEffects(cnsNode, otherNode, GTF_ALL_EFFECT); - break; - } - } - else if (varTypeIsFloating(simdBaseType)) - { - // Handle `(x > NaN) == false` and `(NaN > x) == false` for floating-point types - if (cnsNode->IsVectorNaN(simdBaseType)) - { - int64_t zero = 0; - cnsNode->AsVecCon()->EvaluateBroadcastInPlace(TYP_LONG, zero); - resultNode = gtWrapWithSideEffects(cnsNode, otherNode, GTF_ALL_EFFECT); - break; - } - } - break; - } - - case GT_GE: - { - if (varTypeIsUnsigned(simdBaseType)) - { - // Handle `x >= 0 == true` for unsigned types. - if ((cnsNode == op2) && cnsNode->IsVectorZero()) - { - int64_t allBitsSet = -1; - cnsNode->AsVecCon()->EvaluateBroadcastInPlace(TYP_LONG, allBitsSet); - resultNode = gtWrapWithSideEffects(cnsNode, otherNode, GTF_ALL_EFFECT); - break; - } - } - else if (varTypeIsFloating(simdBaseType)) - { - // Handle `(x >= NaN) == false` and `(NaN >= x) == false` for floating-point types - if (cnsNode->IsVectorNaN(simdBaseType)) - { - int64_t zero = 0; - cnsNode->AsVecCon()->EvaluateBroadcastInPlace(TYP_LONG, zero); - resultNode = gtWrapWithSideEffects(cnsNode, otherNode, GTF_ALL_EFFECT); - break; - } - } - break; - } - - case GT_LT: - { - if (varTypeIsUnsigned(simdBaseType)) - { - // Handle `x < 0 == false` for unsigned types. - if ((cnsNode == op2) && cnsNode->IsVectorZero()) - { - int64_t zero = 0; - cnsNode->AsVecCon()->EvaluateBroadcastInPlace(TYP_LONG, zero); - resultNode = gtWrapWithSideEffects(cnsNode, otherNode, GTF_ALL_EFFECT); - break; - } - } - else if (varTypeIsFloating(simdBaseType)) - { - // Handle `(x < NaN) == false` and `(NaN < x) == false` for floating-point types - if (cnsNode->IsVectorNaN(simdBaseType)) - { - int64_t zero = 0; - cnsNode->AsVecCon()->EvaluateBroadcastInPlace(TYP_LONG, zero); - resultNode = gtWrapWithSideEffects(cnsNode, otherNode, GTF_ALL_EFFECT); - break; - } - } - break; - } - - case GT_LE: - { - if (varTypeIsUnsigned(simdBaseType)) - { - // Handle `0 <= x == true` for unsigned types. - if ((cnsNode == op1) && cnsNode->IsVectorZero()) - { - int64_t allBitsSet = -1; - cnsNode->AsVecCon()->EvaluateBroadcastInPlace(TYP_LONG, allBitsSet); - resultNode = gtWrapWithSideEffects(cnsNode, otherNode, GTF_ALL_EFFECT); - break; - } - } - else if (varTypeIsFloating(simdBaseType)) - { - // Handle `(x <= NaN) == false` and `(NaN <= x) == false` for floating-point types - if (cnsNode->IsVectorNaN(simdBaseType)) - { - int64_t zero = 0; - cnsNode->AsVecCon()->EvaluateBroadcastInPlace(TYP_LONG, zero); - resultNode = gtWrapWithSideEffects(cnsNode, otherNode, GTF_ALL_EFFECT); - break; - } - } - break; - } - - case GT_MUL: - { - if (!varTypeIsFloating(simdBaseType)) - { - // Handle `x * 0 == 0` and `0 * x == 0` - // Not safe for floating-point when x == -0.0, NaN, +Inf, -Inf - if (cnsNode->IsVectorZero()) - { - resultNode = gtWrapWithSideEffects(cnsNode, otherNode, GTF_ALL_EFFECT); - break; - } - } - else - { - // Handle `x * NaN == NaN` and `NaN * x == NaN` - // This is safe for all floats since we do not fault for sNaN - - if (cnsNode->IsVectorNaN(simdBaseType)) - { - resultNode = gtWrapWithSideEffects(cnsNode, otherNode, GTF_ALL_EFFECT); - break; - } - - // We cannot handle `x * 0 == 0` or ` 0 * x == 0` since `-0 * 0 == -0` - // We cannot handle `x * -0 == -0` or `-0 * x == -0` since `-0 * -0 == 0` - } - - // Handle `x * 1 == x` and `1 * x == x` - // This is safe for all floats since we do not fault for sNaN - - if (!cnsNode->IsVectorBroadcast(simdBaseType)) - { - break; - } - - if (cnsNode->AsVecCon()->IsScalarOne(simdBaseType)) - { - resultNode = otherNode; - } - break; - } - - case GT_NE: - { - if (varTypeIsFloating(simdBaseType)) - { - // Handle `(x != NaN) == true` and `(NaN != x) == true` for floating-point types - if (cnsNode->IsVectorNaN(simdBaseType)) - { - int64_t allBitsSet = -1; - cnsNode->AsVecCon()->EvaluateBroadcastInPlace(TYP_LONG, allBitsSet); - resultNode = gtWrapWithSideEffects(cnsNode, otherNode, GTF_ALL_EFFECT); - break; - } - } - break; - } - - case GT_OR: - { - // Handle `x | 0 == x` and `0 | x == x` - if (cnsNode->IsVectorZero()) - { - resultNode = otherNode; - break; - } - - // Handle `x | AllBitsSet == AllBitsSet` and `AllBitsSet | x == AllBitsSet` - if (cnsNode->IsVectorAllBitsSet()) - { - resultNode = gtWrapWithSideEffects(cnsNode, otherNode, GTF_ALL_EFFECT); - } - break; - } - - case GT_ROL: - case GT_ROR: - case GT_LSH: - case GT_RSH: - case GT_RSZ: - { - // Handle `x rol 0 == x` and `0 rol x == 0` - // Handle `x ror 0 == x` and `0 ror x == 0` - // Handle `x << 0 == x` and `0 << x == 0` - // Handle `x >> 0 == x` and `0 >> x == 0` - // Handle `x >>> 0 == x` and `0 >>> x == 0` - - if (cnsNode->IsVectorZero()) - { - if (cnsNode == op2) - { - resultNode = otherNode; - } - else - { - resultNode = gtWrapWithSideEffects(cnsNode, otherNode, GTF_ALL_EFFECT); - } - } - else if (cnsNode->IsIntegralConst(0)) - { - assert(cnsNode == op2); - resultNode = otherNode; - } - break; - } - - case GT_SUB: - { - if (varTypeIsFloating(simdBaseType)) - { - // Handle `x - NaN == NaN` and `NaN - x == NaN` - // This is safe for all floats since we do not fault for sNaN - - if (cnsNode->IsVectorNaN(simdBaseType)) - { - resultNode = gtWrapWithSideEffects(cnsNode, otherNode, GTF_ALL_EFFECT); - break; - } - - // We cannot handle `x - -0 == x` since `-0 - -0 == 0` - } - - // Handle `x - 0 == x` - if ((op2 == cnsNode) && cnsNode->IsVectorZero()) - { - resultNode = otherNode; - } - break; - } - - case GT_XOR: - { - // Handle `x | 0 == x` and `0 | x == x` - if (cnsNode->IsVectorZero()) - { - resultNode = otherNode; - } - break; - } - - default: - { - break; - } - } - - switch (ni) - { -#ifdef TARGET_ARM64 - case NI_AdvSimd_MultiplyByScalar: - case NI_AdvSimd_Arm64_MultiplyByScalar: - { - if (!varTypeIsFloating(simdBaseType)) - { - // Handle `x * 0 == 0` and `0 * x == 0` - // Not safe for floating-point when x == -0.0, NaN, +Inf, -Inf - if (cnsNode == op1) - { - if (cnsNode->IsVectorZero()) - { - resultNode = gtWrapWithSideEffects(cnsNode, otherNode, GTF_ALL_EFFECT); - break; - } - } - else - { - assert(cnsNode == op2); - - if (cnsNode->AsVecCon()->IsScalarZero(simdBaseType)) - { - int64_t val = 0; - - cnsNode->gtType = retType; - cnsNode->AsVecCon()->EvaluateBroadcastInPlace(simdBaseType, val); - - resultNode = gtWrapWithSideEffects(cnsNode, otherNode, GTF_ALL_EFFECT); - break; - } - } - } - else - { - // Handle `x * NaN == NaN` and `NaN * x == NaN` - // This is safe for all floats since we do not fault for sNaN - - if (cnsNode == op1) - { - if (cnsNode->IsVectorNaN(simdBaseType)) - { - resultNode = gtWrapWithSideEffects(cnsNode, otherNode, GTF_ALL_EFFECT); - break; - } - } - else - { - assert(cnsNode == op2); - double val = cnsNode->AsVecCon()->GetElementFloating(simdBaseType, 0); - - if (FloatingPointUtils::isNaN(val)) - { - cnsNode->gtType = retType; - cnsNode->AsVecCon()->EvaluateBroadcastInPlace(simdBaseType, val); - - resultNode = gtWrapWithSideEffects(cnsNode, otherNode, GTF_ALL_EFFECT); - break; - } - } - - // We cannot handle `x * 0 == 0` or ` 0 * x == 0` since `-0 * 0 == -0` - // We cannot handle `x * -0 == -0` or `-0 * x == -0` since `-0 * -0 == 0` - } - - // Handle x * 1 => x, but only if the scalar RHS is <1, ...>. - if ((cnsNode == op2) && cnsNode->AsVecCon()->IsScalarOne(simdBaseType)) - { - resultNode = otherNode; - } - break; - } -#endif - - case NI_Vector128_op_Equality: -#if defined(TARGET_ARM64) - case NI_Vector64_op_Equality: -#elif defined(TARGET_XARCH) - case NI_Vector256_op_Equality: - case NI_Vector512_op_Equality: -#endif // !TARGET_ARM64 && !TARGET_XARCH - { - if (varTypeIsFloating(simdBaseType)) - { - // Handle `(x == NaN) == false` and `(NaN == x) == false` for floating-point types - if (cnsNode->IsVectorNaN(simdBaseType)) - { - resultNode = gtNewIconNode(0, retType); - resultNode = gtWrapWithSideEffects(resultNode, otherNode, GTF_ALL_EFFECT); - break; - } - } - break; - } - - case NI_Vector128_op_Inequality: -#if defined(TARGET_ARM64) - case NI_Vector64_op_Inequality: -#elif defined(TARGET_XARCH) - case NI_Vector256_op_Inequality: - case NI_Vector512_op_Inequality: -#endif // !TARGET_ARM64 && !TARGET_XARCH - { - if (varTypeIsFloating(simdBaseType)) - { - // Handle `(x != NaN) == true` and `(NaN != x) == true` for floating-point types - if (cnsNode->IsVectorNaN(simdBaseType)) - { - resultNode = gtNewIconNode(1, retType); - resultNode = gtWrapWithSideEffects(resultNode, otherNode, GTF_ALL_EFFECT); - break; - } - } - break; - } - - default: - { - break; - } - } - } - - if (op3 != nullptr) - { - switch (ni) - { -#if defined(TARGET_XARCH) - case NI_Vector128_ConditionalSelect: - case NI_Vector256_ConditionalSelect: - case NI_Vector512_ConditionalSelect: -#elif defined(TARGET_ARM64) - case NI_AdvSimd_BitwiseSelect: - case NI_Sve_ConditionalSelect: -#endif - { - assert(!varTypeIsMask(retType)); - - if (cnsNode != op1) - { - break; - } - - if (op1->IsVectorAllBitsSet() || op1->IsMaskAllBitsSet()) - { - if ((op3->gtFlags & GTF_SIDE_EFFECT) != 0) - { - // op3 has side effects, this would require us to append a new statement - // to ensure that it isn't lost, which isn't safe to do from the general - // purpose handler here. We'll recognize this and mark it in VN instead - break; - } - - // op3 has no side effects, so we can return op2 directly - return op2; - } - - if (op1->IsVectorZero()) - { - return gtWrapWithSideEffects(op3, op2, GTF_ALL_EFFECT); - } - - if (op2->IsCnsVec() && op3->IsCnsVec()) - { - // op2 = op2 & op1 - op2->AsVecCon()->EvaluateBinaryInPlace(GT_AND, false, simdBaseType, op1->AsVecCon()); - - // op3 = op2 & ~op1 - op3->AsVecCon()->EvaluateBinaryInPlace(GT_AND_NOT, false, simdBaseType, op1->AsVecCon()); - - // op2 = op2 | op3 - op2->AsVecCon()->EvaluateBinaryInPlace(GT_OR, false, simdBaseType, op3->AsVecCon()); - - resultNode = op2; - } - break; - } - - default: - { - break; - } - } - } - - if (varTypeIsMask(retType) && !varTypeIsMask(resultNode)) - { - resultNode = gtNewSimdCvtVectorToMaskNode(retType, resultNode, simdBaseJitType, simdSize); - return gtFoldExprHWIntrinsic(resultNode->AsHWIntrinsic()); - } - - if (resultNode != tree) - { - resultNode->SetMorphed(this); - if (resultNode->OperIs(GT_COMMA)) - { - resultNode->AsOp()->gtGetOp2()->SetMorphed(this); - } - - if (resultNode->OperIsConst()) - { - fgUpdateConstTreeValueNumber(resultNode); - - // Make sure no side effect flags are set on this constant node. - resultNode->gtFlags &= ~GTF_ALL_EFFECT; - } - } - return resultNode; -} -#endif // FEATURE_HW_INTRINSICS - -//------------------------------------------------------------------------ -// gtCanSkipCovariantStoreCheck: see if storing a ref type value to an array -// can skip the array store covariance check. -// -// Arguments: -// value -- tree producing the value to store -// array -- tree representing the array to store to -// -// Returns: -// true if the store does not require a covariance check. -// -bool Compiler::gtCanSkipCovariantStoreCheck(GenTree* value, GenTree* array) -{ - // We should only call this when optimizing. - assert(opts.OptimizationEnabled()); - - // Check for store to same array, ie. arrLcl[i] = arrLcl[j] - if (value->OperIs(GT_IND) && value->AsIndir()->Addr()->OperIs(GT_INDEX_ADDR) && array->OperIs(GT_LCL_VAR)) - { - GenTree* valueArray = value->AsIndir()->Addr()->AsIndexAddr()->Arr(); - if (valueArray->OperIs(GT_LCL_VAR)) - { - unsigned valueArrayLcl = valueArray->AsLclVar()->GetLclNum(); - unsigned arrayLcl = array->AsLclVar()->GetLclNum(); - if ((valueArrayLcl == arrayLcl) && !lvaGetDesc(arrayLcl)->IsAddressExposed()) - { - JITDUMP("\nstelem of ref from same array: skipping covariant store check\n"); - return true; - } - } - } - - // Check for store of NULL. - if (value->OperIs(GT_CNS_INT)) - { - assert(value->gtType == TYP_REF); - if (value->AsIntCon()->gtIconVal == 0) - { - JITDUMP("\nstelem of null: skipping covariant store check\n"); - return true; - } - // Non-0 const refs can only occur with frozen objects - assert(value->IsIconHandle(GTF_ICON_OBJ_HDL)); - assert(doesMethodHaveFrozenObjects() || - (compIsForInlining() && impInlineInfo->InlinerCompiler->doesMethodHaveFrozenObjects())); - } - - // Try and get a class handle for the array - if (!value->TypeIs(TYP_REF)) - { - return false; - } - - bool arrayIsExact = false; - bool arrayIsNonNull = false; - CORINFO_CLASS_HANDLE arrayHandle = gtGetClassHandle(array, &arrayIsExact, &arrayIsNonNull); - - if (arrayHandle == NO_CLASS_HANDLE) - { - return false; - } - - // There are some methods in corelib where we're storing to an array but the IL - // doesn't reflect this (see SZArrayHelper). Avoid. - DWORD attribs = info.compCompHnd->getClassAttribs(arrayHandle); - if ((attribs & CORINFO_FLG_ARRAY) == 0) - { - return false; - } - - CORINFO_CLASS_HANDLE arrayElementHandle = nullptr; - CorInfoType arrayElemType = info.compCompHnd->getChildType(arrayHandle, &arrayElementHandle); - - // Verify array type handle is really an array of ref type - assert(arrayElemType == CORINFO_TYPE_CLASS); - - // Check for exactly object[] - if (arrayIsExact && (arrayElementHandle == impGetObjectClass())) - { - JITDUMP("\nstelem to (exact) object[]: skipping covariant store check\n"); - return true; - } - - const bool arrayTypeIsSealed = info.compCompHnd->isExactType(arrayElementHandle); - - if ((!arrayIsExact && !arrayTypeIsSealed) || (arrayElementHandle == NO_CLASS_HANDLE)) - { - // Bail out if we don't know array's exact type - return false; - } - - bool valueIsExact = false; - bool valueIsNonNull = false; - CORINFO_CLASS_HANDLE valueHandle = gtGetClassHandle(value, &valueIsExact, &valueIsNonNull); - - // Array's type is sealed and equals to value's type - if (arrayTypeIsSealed && (valueHandle == arrayElementHandle)) - { - JITDUMP("\nstelem to T[] with T exact: skipping covariant store check\n"); - return true; - } - - // Array's type is not sealed but we know its exact type - if (arrayIsExact && (valueHandle != NO_CLASS_HANDLE) && - (info.compCompHnd->compareTypesForCast(valueHandle, arrayElementHandle) == TypeCompareState::Must)) - { - JITDUMP("\nstelem to T[] with T exact: skipping covariant store check\n"); - return true; - } - - return false; -} - -#if defined(DEBUG) -//------------------------------------------------------------------------ -// SetMorphed: mark a node as having been morphed -// -// Arguments: -// compiler - compiler instance -// doChildren - recursive mark child nodes -// -// Notes: -// Does nothing outside of global morph. -// -// Useful for morph post-order expansions / optimizations. -// -// Use care when invoking this on an assignment (or when doChildren is true, -// on trees containing assignments) as those usually will also require -// local assertion updates. -// -void GenTree::SetMorphed(Compiler* compiler, bool doChildren /* = false */) -{ - if (!compiler->fgGlobalMorph) - { - return; - } - - struct Visitor : GenTreeVisitor - { - enum - { - DoPostOrder = true, - }; - - Visitor(Compiler* comp) - : GenTreeVisitor(comp) - { - } - - fgWalkResult PostOrderVisit(GenTree** use, GenTree* user) - { - GenTree* const node = *use; - if (!node->WasMorphed()) - { - node->gtDebugFlags |= GTF_DEBUG_NODE_MORPHED; - node->gtMorphCount++; - } - return Compiler::WALK_CONTINUE; - } - }; - - if (doChildren) - { - Visitor v(compiler); - GenTree* node = this; - v.WalkTree(&node, nullptr); - } - else if (!WasMorphed()) - { - gtDebugFlags |= GTF_DEBUG_NODE_MORPHED; - gtMorphCount++; - } -} -#endif + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + if ((info.compCompHnd->getExactClasses(clsHnd, 0, nullptr) == 0) && JitConfig.JitEnableExactDevirtualization()) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + if ((val == 0) && op->TypeIs(TYP_REF) && IsTargetAbi(CORINFO_NATIVEAOT_ABI) && + JitConfig.JitEnableExactDevirtualization()) \ No newline at end of file