Add peephole optimization to reduce MaskToVec and VecToMask conversions

src/coreclr/jit/assertionprop.cpp

@@ -5671,6 +5671,109 @@ GenTree* Compiler::optAssertionProp_BndsChk(ASSERT_VALARG_TP assertions,
     return nullptr;
 }
 
+/*****************************************************************************
+ *
+ *  Given a tree where there exists a Vec2Mask(NOT(LCL_VAR)) statement, and LCL_VAR is
+ *  defined as some Mask2Vec conversion: LCL_VAR = Mask2Vec(m), we reduce the tree to
+ *  NOT(m). 
+ */
+
+#if defined(FEATURE_HW_INTRINSICS)
+GenTree* Compiler::optAssertionProp_FoldVecMaskConv(ASSERT_VALARG_TP assertions, GenTreeHWIntrinsic* tree, Statement* stmt) 
+{
+    #if defined(FEATURE_MASKED_HW_INTRINSICS) && defined(TARGET_XARCH)
+    if (!tree->OperIsConvertVectorToMask())
+    {
+        return nullptr;
+    }
+
+    if (optLocalAssertionProp)
+    {
+        return nullptr;
+    }
+
+    GenTree* op = tree->Op(1);
+    bool     hasIntermediateNot = false;
+    GenTree* localOp            = op;
+
+    if (op->OperIsHWIntrinsic())
+    {
+        GenTreeHWIntrinsic* hwOp  = op->AsHWIntrinsic();
+        bool isScalar;
+        genTreeOps          oper  = hwOp->GetOperForHWIntrinsicId(&isScalar);
+
+        if ((oper == GT_XOR) && hwOp->Op(2)->IsVectorAllBitsSet())
+        {
+            hasIntermediateNot = true;
+            localOp            = hwOp->Op(1);
+        }
+    }
+
+
+    if (!localOp->OperIs(GT_LCL_VAR))
+    {
+        return nullptr;
+    }
+
+    GenTreeLclVarCommon* use    = localOp->AsLclVarCommon();
+    unsigned             ssaNum = use->GetSsaNum();
+    if (ssaNum == SsaConfig::RESERVED_SSA_NUM)
+    {
+        return nullptr;
+    }
+
+    LclSsaVarDsc*        ssaDef   = lvaGetDesc(use)->GetPerSsaData(ssaNum);
+    GenTreeLclVarCommon* defStore = ssaDef->GetDefNode();
+    if ((defStore == nullptr) || !defStore->OperIs(GT_STORE_LCL_VAR))
+    {
+        return nullptr;
+    }
+
+    GenTree* stored = defStore->Data();
+
+    if (!stored->OperIsConvertMaskToVector())
+    {
+        return nullptr;
+    }
+
+
+    GenTreeHWIntrinsic* innerCvt      = stored->AsHWIntrinsic();
+    var_types           innerBaseType = innerCvt->GetSimdBaseType();
+
+    if (genTypeSize(innerBaseType) != genTypeSize(tree->GetSimdBaseType()))
+    {
+        return nullptr;
+    }
+
+    if (innerBaseType != TYP_INT && innerBaseType != TYP_UINT)
+    {
+        return nullptr;
+    }
+
+
+    GenTree* innerVal = innerCvt->Op(1);
+    GenTree* result = gtCloneExpr(innerVal);
+
+
+    if (hasIntermediateNot)
+    {
+        NamedIntrinsic notId = NI_AVX512_NotMask;
+        result = gtNewSimdHWIntrinsicNode(TYP_MASK,
+                                        result, notId,
+                                        innerBaseType, tree->GetSimdSize());
+
+    }
+
+    JITDUMP("optAssertionProp_FoldVecMaskConv: folding mask/vector round-trip through V%02d\n",
+            use->GetLclNum());
+
+    return optAssertionProp_Update(result, tree, stmt);
+#else
+    return nullptr;
+#endif // FEATURE_MASKED_HW_INTRINSICS && TARGET_XARCH
+}
+#endif // FEATURE_HW_INTRINSICS
+
 /*****************************************************************************
  *
  *  Called when we have a successfully performed an assertion prop. We have
@@ -5837,6 +5940,12 @@ GenTree* Compiler::optAssertionProp(ASSERT_VALARG_TP assertions, GenTree* tree,
             }
             return nullptr;
 
+#if defined(FEATURE_HW_INTRINSICS)
+        case GT_HWINTRINSIC:
+            return optAssertionProp_FoldVecMaskConv(assertions, tree->AsHWIntrinsic(), stmt);
+#endif // FEATURE_HW_INTRINSICS
+
+
         default:
             return nullptr;
     }

src/coreclr/jit/compiler.h

@@ -9245,6 +9245,9 @@ class Compiler
                                           GenTree*         tree,
                                           Statement*       stmt,
                                           BasicBlock*      block);
+#if defined(FEATURE_HW_INTRINSICS)
+    GenTree* optAssertionProp_FoldVecMaskConv(ASSERT_VALARG_TP assertions, GenTreeHWIntrinsic* tree, Statement* stmt);
+#endif // FEATURE_HW_INTRINSICS
     GenTree* optAssertionPropLocal_RelOp(ASSERT_VALARG_TP assertions, GenTree* tree, Statement* stmt);
     GenTree* optAssertionProp_Update(GenTree* newTree, GenTree* tree, Statement* stmt);
     GenTree* optNonNullAssertionProp_Call(ASSERT_VALARG_TP assertions, GenTreeCall* call);