Skip to content

[RISCV] Add intrinsics for strided segment stores with fixed vectors #152038

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open

[RISCV] Add intrinsics for strided segment stores with fixed vectors #152038

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
bf4b7f1
[RISCV] Add intrinsics for strided segment stores with fixed vectors
mshockwave Aug 4, 2025
1e07d84
fixup! [RISCV] Add intrinsics for strided segment stores with fixed v…
mshockwave Aug 4, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions llvm/include/llvm/IR/IntrinsicsRISCV.td
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1736,6 +1736,17 @@ let TargetPrefix = "riscv" in {
[llvm_anyptr_ty, LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
llvm_anyint_ty]),
[NoCapture<ArgIndex<nf>>, IntrWriteMem]>;

// Input: (<stored values>..., pointer, offset, mask, vl)
def int_riscv_sseg # nf # _store_mask
: DefaultAttrsIntrinsic<[],
!listconcat([llvm_anyvector_ty],
!listsplat(LLVMMatchType<0>,
!add(nf, -1)),
[llvm_anyptr_ty, llvm_anyint_ty,
LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
llvm_anyint_ty]),
[NoCapture<ArgIndex<nf>>, IntrWriteMem]>;
}

} // TargetPrefix = "riscv"
Expand Down
160 changes: 110 additions & 50 deletions llvm/lib/Target/RISCV/RISCVISelLowering.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1839,6 +1839,17 @@ bool RISCVTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
return SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 3,
/*IsStore*/ true,
/*IsUnitStrided*/ false, /*UsePtrVal*/ true);
case Intrinsic::riscv_sseg2_store_mask:
case Intrinsic::riscv_sseg3_store_mask:
case Intrinsic::riscv_sseg4_store_mask:
case Intrinsic::riscv_sseg5_store_mask:
case Intrinsic::riscv_sseg6_store_mask:
case Intrinsic::riscv_sseg7_store_mask:
case Intrinsic::riscv_sseg8_store_mask:
// Operands are (vec, ..., vec, ptr, offset, mask, vl)
return SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 4,
/*IsStore*/ true,
/*IsUnitStrided*/ false, /*UsePtrVal*/ true);
case Intrinsic::riscv_vlm:
return SetRVVLoadStoreInfo(/*PtrOp*/ 0,
/*IsStore*/ false,
Expand Down Expand Up @@ -11077,69 +11088,118 @@ SDValue RISCVTargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
return lowerVectorIntrinsicScalars(Op, DAG, Subtarget);
}

SDValue RISCVTargetLowering::LowerINTRINSIC_VOID(SDValue Op,
SelectionDAG &DAG) const {
unsigned IntNo = Op.getConstantOperandVal(1);
static SDValue
lowerFixedVectorSegStoreIntrinsics(unsigned IntNo, SDValue Op,
const RISCVSubtarget &Subtarget,
SelectionDAG &DAG) {
bool IsStrided;
switch (IntNo) {
default:
break;
case Intrinsic::riscv_seg2_store_mask:
case Intrinsic::riscv_seg3_store_mask:
case Intrinsic::riscv_seg4_store_mask:
case Intrinsic::riscv_seg5_store_mask:
case Intrinsic::riscv_seg6_store_mask:
case Intrinsic::riscv_seg7_store_mask:
case Intrinsic::riscv_seg8_store_mask: {
SDLoc DL(Op);
static const Intrinsic::ID VssegInts[] = {
Intrinsic::riscv_vsseg2_mask, Intrinsic::riscv_vsseg3_mask,
Intrinsic::riscv_vsseg4_mask, Intrinsic::riscv_vsseg5_mask,
Intrinsic::riscv_vsseg6_mask, Intrinsic::riscv_vsseg7_mask,
Intrinsic::riscv_vsseg8_mask};
case Intrinsic::riscv_seg8_store_mask:
IsStrided = false;
break;
case Intrinsic::riscv_sseg2_store_mask:
case Intrinsic::riscv_sseg3_store_mask:
case Intrinsic::riscv_sseg4_store_mask:
case Intrinsic::riscv_sseg5_store_mask:
case Intrinsic::riscv_sseg6_store_mask:
case Intrinsic::riscv_sseg7_store_mask:
case Intrinsic::riscv_sseg8_store_mask:
IsStrided = true;
break;
default:
llvm_unreachable("unexpected intrinsic ID");
}

// Operands: (chain, int_id, vec*, ptr, mask, vl)
unsigned NF = Op->getNumOperands() - 5;
assert(NF >= 2 && NF <= 8 && "Unexpected seg number");
MVT XLenVT = Subtarget.getXLenVT();
MVT VT = Op->getOperand(2).getSimpleValueType();
MVT ContainerVT = getContainerForFixedLengthVector(VT);
unsigned Sz = NF * ContainerVT.getVectorMinNumElements() *
ContainerVT.getScalarSizeInBits();
EVT VecTupTy = MVT::getRISCVVectorTupleVT(Sz, NF);
SDLoc DL(Op);
static const Intrinsic::ID VssegInts[] = {
Intrinsic::riscv_vsseg2_mask, Intrinsic::riscv_vsseg3_mask,
Intrinsic::riscv_vsseg4_mask, Intrinsic::riscv_vsseg5_mask,
Intrinsic::riscv_vsseg6_mask, Intrinsic::riscv_vsseg7_mask,
Intrinsic::riscv_vsseg8_mask};
static const Intrinsic::ID VsssegInts[] = {
Intrinsic::riscv_vssseg2_mask, Intrinsic::riscv_vssseg3_mask,
Intrinsic::riscv_vssseg4_mask, Intrinsic::riscv_vssseg5_mask,
Intrinsic::riscv_vssseg6_mask, Intrinsic::riscv_vssseg7_mask,
Intrinsic::riscv_vssseg8_mask};

// Operands: (chain, int_id, vec*, ptr, mask, vl) or
// (chain, int_id, vec*, ptr, stride, mask, vl)
unsigned NF = Op->getNumOperands() - (IsStrided ? 6 : 5);
assert(NF >= 2 && NF <= 8 && "Unexpected seg number");
MVT XLenVT = Subtarget.getXLenVT();
MVT VT = Op->getOperand(2).getSimpleValueType();
MVT ContainerVT = ::getContainerForFixedLengthVector(DAG, VT, Subtarget);
unsigned Sz = NF * ContainerVT.getVectorMinNumElements() *
ContainerVT.getScalarSizeInBits();
EVT VecTupTy = MVT::getRISCVVectorTupleVT(Sz, NF);

SDValue VL = Op.getOperand(Op.getNumOperands() - 1);
SDValue Mask = Op.getOperand(Op.getNumOperands() - 2);
MVT MaskVT = Mask.getSimpleValueType();
MVT MaskContainerVT =
::getContainerForFixedLengthVector(DAG, MaskVT, Subtarget);
Mask = convertToScalableVector(MaskContainerVT, Mask, DAG, Subtarget);
SDValue VL = Op.getOperand(Op.getNumOperands() - 1);
SDValue Mask = Op.getOperand(Op.getNumOperands() - 2);
MVT MaskVT = Mask.getSimpleValueType();
MVT MaskContainerVT =
::getContainerForFixedLengthVector(DAG, MaskVT, Subtarget);
Mask = convertToScalableVector(MaskContainerVT, Mask, DAG, Subtarget);

SDValue IntID = DAG.getTargetConstant(VssegInts[NF - 2], DL, XLenVT);
SDValue Ptr = Op->getOperand(NF + 2);
SDValue IntID = DAG.getTargetConstant(
IsStrided ? VsssegInts[NF - 2] : VssegInts[NF - 2], DL, XLenVT);
SDValue Ptr = Op->getOperand(NF + 2);

auto *FixedIntrinsic = cast<MemIntrinsicSDNode>(Op);
auto *FixedIntrinsic = cast<MemIntrinsicSDNode>(Op);

SDValue StoredVal = DAG.getUNDEF(VecTupTy);
for (unsigned i = 0; i < NF; i++)
StoredVal = DAG.getNode(
RISCVISD::TUPLE_INSERT, DL, VecTupTy, StoredVal,
convertToScalableVector(
ContainerVT, FixedIntrinsic->getOperand(2 + i), DAG, Subtarget),
DAG.getTargetConstant(i, DL, MVT::i32));
SDValue StoredVal = DAG.getUNDEF(VecTupTy);
for (unsigned i = 0; i < NF; i++)
StoredVal = DAG.getNode(
RISCVISD::TUPLE_INSERT, DL, VecTupTy, StoredVal,
convertToScalableVector(ContainerVT, FixedIntrinsic->getOperand(2 + i),
DAG, Subtarget),
DAG.getTargetConstant(i, DL, MVT::i32));

SmallVector<SDValue, 10> Ops = {
FixedIntrinsic->getChain(),
IntID,
StoredVal,
Ptr,
Mask,
VL,
DAG.getTargetConstant(Log2_64(VT.getScalarSizeInBits()), DL, XLenVT)};
// Insert the stride operand.
if (IsStrided)
Ops.insert(std::next(Ops.begin(), 4),
Op.getOperand(Op.getNumOperands() - 3));

return DAG.getMemIntrinsicNode(
ISD::INTRINSIC_VOID, DL, DAG.getVTList(MVT::Other), Ops,
FixedIntrinsic->getMemoryVT(), FixedIntrinsic->getMemOperand());
}

SDValue RISCVTargetLowering::LowerINTRINSIC_VOID(SDValue Op,
SelectionDAG &DAG) const {
unsigned IntNo = Op.getConstantOperandVal(1);
switch (IntNo) {
default:
break;
case Intrinsic::riscv_seg2_store_mask:
case Intrinsic::riscv_seg3_store_mask:
case Intrinsic::riscv_seg4_store_mask:
case Intrinsic::riscv_seg5_store_mask:
case Intrinsic::riscv_seg6_store_mask:
case Intrinsic::riscv_seg7_store_mask:
case Intrinsic::riscv_seg8_store_mask:
case Intrinsic::riscv_sseg2_store_mask:
case Intrinsic::riscv_sseg3_store_mask:
case Intrinsic::riscv_sseg4_store_mask:
case Intrinsic::riscv_sseg5_store_mask:
case Intrinsic::riscv_sseg6_store_mask:
case Intrinsic::riscv_sseg7_store_mask:
case Intrinsic::riscv_sseg8_store_mask:
return lowerFixedVectorSegStoreIntrinsics(IntNo, Op, Subtarget, DAG);

SDValue Ops[] = {
FixedIntrinsic->getChain(),
IntID,
StoredVal,
Ptr,
Mask,
VL,
DAG.getTargetConstant(Log2_64(VT.getScalarSizeInBits()), DL, XLenVT)};

return DAG.getMemIntrinsicNode(
ISD::INTRINSIC_VOID, DL, DAG.getVTList(MVT::Other), Ops,
FixedIntrinsic->getMemoryVT(), FixedIntrinsic->getMemOperand());
}
case Intrinsic::riscv_sf_vc_xv_se:
return getVCIXISDNodeVOID(Op, DAG, RISCVISD::SF_VC_XV_SE);
case Intrinsic::riscv_sf_vc_iv_se:
Expand Down
72 changes: 72 additions & 0 deletions llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ssegN-store.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc -mtriple=riscv64 -mattr=+v -verify-machineinstrs < %s | FileCheck %s

define void @store_factor2(<8 x i8> %v0, <8 x i8> %v1, ptr %ptr, i64 %stride) {
; CHECK-LABEL: store_factor2:
; CHECK: # %bb.0:
; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
; CHECK-NEXT: vssseg2e8.v v8, (a0), a1
; CHECK-NEXT: ret
call void @llvm.riscv.sseg2.store.mask.v8i8.i64.i64(<8 x i8> %v0, <8 x i8> %v1, ptr %ptr, i64 %stride, <8 x i1> splat (i1 true), i64 8)
ret void
}

define void @store_factor3(<8 x i8> %v0, <8 x i8> %v1, <8 x i8> %v2, ptr %ptr, i64 %stride) {
; CHECK-LABEL: store_factor3:
; CHECK: # %bb.0:
; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
; CHECK-NEXT: vssseg3e8.v v8, (a0), a1
; CHECK-NEXT: ret
call void @llvm.riscv.sseg3.store.mask.v8i8.i64.i64(<8 x i8> %v0, <8 x i8> %v1, <8 x i8> %v2, ptr %ptr, i64 %stride, <8 x i1> splat (i1 true), i64 8)
ret void
}

define void @store_factor4(<8 x i8> %v0, <8 x i8> %v1, <8 x i8> %v2, <8 x i8> %v3, ptr %ptr, i64 %stride) {
; CHECK-LABEL: store_factor4:
; CHECK: # %bb.0:
; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
; CHECK-NEXT: vssseg4e8.v v8, (a0), a1
; CHECK-NEXT: ret
call void @llvm.riscv.sseg4.store.mask.v8i8.i64.i64(<8 x i8> %v0, <8 x i8> %v1, <8 x i8> %v2, <8 x i8> %v3, ptr %ptr, i64 %stride, <8 x i1> splat (i1 true), i64 8)
ret void
}

define void @store_factor5(<8 x i8> %v0, <8 x i8> %v1, <8 x i8> %v2, <8 x i8> %v3, <8 x i8> %v4, ptr %ptr, i64 %stride) {
; CHECK-LABEL: store_factor5:
; CHECK: # %bb.0:
; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
; CHECK-NEXT: vssseg5e8.v v8, (a0), a1
; CHECK-NEXT: ret
call void @llvm.riscv.sseg5.store.mask.v8i8.i64.i64(<8 x i8> %v0, <8 x i8> %v1, <8 x i8> %v2, <8 x i8> %v3, <8 x i8> %v4, ptr %ptr, i64 %stride, <8 x i1> splat (i1 true), i64 8)
ret void
}

define void @store_factor6(<8 x i8> %v0, <8 x i8> %v1, <8 x i8> %v2, <8 x i8> %v3, <8 x i8> %v4, <8 x i8> %v5, ptr %ptr, i64 %stride) {
; CHECK-LABEL: store_factor6:
; CHECK: # %bb.0:
; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
; CHECK-NEXT: vssseg6e8.v v8, (a0), a1
; CHECK-NEXT: ret
call void @llvm.riscv.sseg6.store.mask.v8i8.i64.i64(<8 x i8> %v0, <8 x i8> %v1, <8 x i8> %v2, <8 x i8> %v3, <8 x i8> %v4, <8 x i8> %v5, ptr %ptr, i64 %stride, <8 x i1> splat (i1 true), i64 8)
ret void
}

define void @store_factor7(<8 x i8> %v0, <8 x i8> %v1, <8 x i8> %v2, <8 x i8> %v3, <8 x i8> %v4, <8 x i8> %v5, <8 x i8> %v6, ptr %ptr, i64 %stride) {
; CHECK-LABEL: store_factor7:
; CHECK: # %bb.0:
; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
; CHECK-NEXT: vssseg7e8.v v8, (a0), a1
; CHECK-NEXT: ret
call void @llvm.riscv.sseg7.store.mask.v8i8.i64.i64(<8 x i8> %v0, <8 x i8> %v1, <8 x i8> %v2, <8 x i8> %v3, <8 x i8> %v4, <8 x i8> %v5, <8 x i8> %v6, ptr %ptr, i64 %stride, <8 x i1> splat (i1 true), i64 8)
ret void
}

define void @store_factor8(<8 x i8> %v0, <8 x i8> %v1, <8 x i8> %v2, <8 x i8> %v3, <8 x i8> %v4, <8 x i8> %v5, <8 x i8> %v6, <8 x i8> %v7, ptr %ptr, i64 %stride) {
; CHECK-LABEL: store_factor8:
; CHECK: # %bb.0:
; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
; CHECK-NEXT: vssseg8e8.v v8, (a0), a1
; CHECK-NEXT: ret
call void @llvm.riscv.sseg8.store.mask.v8i8.i64.i64(<8 x i8> %v0, <8 x i8> %v1, <8 x i8> %v2, <8 x i8> %v3, <8 x i8> %v4, <8 x i8> %v5, <8 x i8> %v6, <8 x i8> %v7, ptr %ptr, i64 %stride, <8 x i1> splat (i1 true), i64 8)
ret void
}