Skip to content

Commit

Permalink
riscv64: Implement SIMD fcmp (bytecodealliance#6643)
Browse files Browse the repository at this point in the history 
* riscv64: Add float vector mask instructions

* riscv64: Add some vector mask bitwise instructions

* riscv64: Implement SIMD `fcmp`

* cranelift: Add SIMD `fcmp` testsuite
  • Loading branch information
@afonso360
afonso360 authored Jun 27, 2023
1 parent 4c23880 commit 3cab644
Show file tree
Hide file tree
Showing 34 changed files with 5,368 additions and 75 deletions.
2 changes: 0 additions & 2 deletions build.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -234,11 +234,9 @@ fn ignore(testsuite: &str, testname: &str, strategy: &str) -> bool {
"issue_3327_bnot_lowering",
"simd_conversions",
"simd_f32x4",
"simd_f32x4_cmp",
"simd_f32x4_pmin_pmax",
"simd_f32x4_rounding",
"simd_f64x2",
"simd_f64x2_cmp",
"simd_f64x2_pmin_pmax",
"simd_f64x2_rounding",
"simd_i32x4_trunc_sat_f32x4",
Expand Down
3 changes: 3 additions & 0 deletions cranelift/codegen/src/isa/riscv64/inst/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1744,6 +1744,9 @@ impl Inst {
(VecAluOpRRR::VfsgnjnVV, vs2, vs1) if vs2 == vs1 => {
format!("vfneg.v {vd_s},{vs2_s}{mask} {vstate}")
}
(VecAluOpRRR::VmnandMM, vs2, vs1) if vs2 == vs1 => {
format!("vmnot.m {vd_s},{vs2_s}{mask} {vstate}")
}
_ => format!("{op} {vd_s},{vs2_s},{vs1_s}{mask} {vstate}"),
}
}
Expand Down
52 changes: 41 additions & 11 deletions cranelift/codegen/src/isa/riscv64/inst/vector.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -357,14 +357,30 @@ impl VecAluOpRRR {
VecAluOpRRR::VwaddWV | VecAluOpRRR::VwaddWX => 0b110101,
VecAluOpRRR::VwsubuWV | VecAluOpRRR::VwsubuWX => 0b110110,
VecAluOpRRR::VwsubWV | VecAluOpRRR::VwsubWX => 0b110111,
VecAluOpRRR::VmseqVV | VecAluOpRRR::VmseqVX => 0b011000,
VecAluOpRRR::VmsneVV | VecAluOpRRR::VmsneVX => 0b011001,
VecAluOpRRR::VmsltuVV | VecAluOpRRR::VmsltuVX => 0b011010,
VecAluOpRRR::VmsltVV | VecAluOpRRR::VmsltVX => 0b011011,
VecAluOpRRR::VmsleuVV | VecAluOpRRR::VmsleuVX => 0b011100,
VecAluOpRRR::VmsleVV | VecAluOpRRR::VmsleVX => 0b011101,
VecAluOpRRR::VmsgtuVX => 0b011110,
VecAluOpRRR::VmsgtVX => 0b011111,
VecAluOpRRR::VmseqVV
| VecAluOpRRR::VmseqVX
| VecAluOpRRR::VmfeqVV
| VecAluOpRRR::VmfeqVF => 0b011000,
VecAluOpRRR::VmsneVV
| VecAluOpRRR::VmsneVX
| VecAluOpRRR::VmfleVV
| VecAluOpRRR::VmfleVF
| VecAluOpRRR::VmandMM => 0b011001,
VecAluOpRRR::VmsltuVV | VecAluOpRRR::VmsltuVX | VecAluOpRRR::VmorMM => 0b011010,
VecAluOpRRR::VmsltVV
| VecAluOpRRR::VmsltVX
| VecAluOpRRR::VmfltVV
| VecAluOpRRR::VmfltVF => 0b011011,
VecAluOpRRR::VmsleuVV
| VecAluOpRRR::VmsleuVX
| VecAluOpRRR::VmfneVV
| VecAluOpRRR::VmfneVF => 0b011100,
VecAluOpRRR::VmsleVV
| VecAluOpRRR::VmsleVX
| VecAluOpRRR::VmfgtVF
| VecAluOpRRR::VmnandMM => 0b011101,
VecAluOpRRR::VmsgtuVX | VecAluOpRRR::VmnorMM => 0b011110,
VecAluOpRRR::VmsgtVX | VecAluOpRRR::VmfgeVF => 0b011111,
}
}

Expand Down Expand Up @@ -408,7 +424,11 @@ impl VecAluOpRRR {
| VecAluOpRRR::VmulhuVV
| VecAluOpRRR::VredmaxuVS
| VecAluOpRRR::VredminuVS
| VecAluOpRRR::VcompressVM => VecOpCategory::OPMVV,
| VecAluOpRRR::VcompressVM
| VecAluOpRRR::VmandMM
| VecAluOpRRR::VmorMM
| VecAluOpRRR::VmnandMM
| VecAluOpRRR::VmnorMM => VecOpCategory::OPMVV,
VecAluOpRRR::VwaddVX
| VecAluOpRRR::VwadduVX
| VecAluOpRRR::VwadduWX
Expand Down Expand Up @@ -453,14 +473,24 @@ impl VecAluOpRRR {
| VecAluOpRRR::VfsubVV
| VecAluOpRRR::VfmulVV
| VecAluOpRRR::VfdivVV
| VecAluOpRRR::VfsgnjnVV => VecOpCategory::OPFVV,
| VecAluOpRRR::VfsgnjnVV
| VecAluOpRRR::VmfeqVV
| VecAluOpRRR::VmfneVV
| VecAluOpRRR::VmfltVV
| VecAluOpRRR::VmfleVV => VecOpCategory::OPFVV,
VecAluOpRRR::VfaddVF
| VecAluOpRRR::VfsubVF
| VecAluOpRRR::VfrsubVF
| VecAluOpRRR::VfmulVF
| VecAluOpRRR::VfdivVF
| VecAluOpRRR::VfrdivVF
| VecAluOpRRR::VfmergeVFM => VecOpCategory::OPFVF,
| VecAluOpRRR::VfmergeVFM
| VecAluOpRRR::VmfeqVF
| VecAluOpRRR::VmfneVF
| VecAluOpRRR::VmfltVF
| VecAluOpRRR::VmfleVF
| VecAluOpRRR::VmfgtVF
| VecAluOpRRR::VmfgeVF => VecOpCategory::OPFVF,
}
}

Expand Down
232 changes: 232 additions & 0 deletions cranelift/codegen/src/isa/riscv64/inst_vector.isle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,14 @@
(VmsltVV)
(VmsleuVV)
(VmsleVV)
(VmfeqVV)
(VmfneVV)
(VmfltVV)
(VmfleVV)
(VmandMM)
(VmorMM)
(VmnandMM)
(VmnorMM)


;; Vector-Scalar Opcodes
Expand Down Expand Up @@ -184,6 +192,12 @@
(VmsleVX)
(VmsgtuVX)
(VmsgtVX)
(VmfeqVF)
(VmfneVF)
(VmfltVF)
(VmfleVF)
(VmfgtVF)
(VmfgeVF)
))


Expand Down Expand Up @@ -1111,6 +1125,66 @@
(decl rv_vmsge_vv (VReg VReg VecOpMasking VState) VReg)
(rule (rv_vmsge_vv vs2 vs1 mask vstate) (rv_vmsle_vv vs1 vs2 mask vstate))

;; Helper for emitting the `vmfeq.vv` (Vector Mask Set If Float Equal) instruction.
(decl rv_vmfeq_vv (VReg VReg VecOpMasking VState) VReg)
(rule (rv_vmfeq_vv vs2 vs1 mask vstate)
(vec_alu_rrr (VecAluOpRRR.VmfeqVV) vs2 vs1 mask vstate))

;; Helper for emitting the `vmfeq.vf` (Vector Mask Set If Float Equal) instruction.
(decl rv_vmfeq_vf (VReg FReg VecOpMasking VState) VReg)
(rule (rv_vmfeq_vf vs2 vs1 mask vstate)
(vec_alu_rrr (VecAluOpRRR.VmfeqVF) vs2 vs1 mask vstate))

;; Helper for emitting the `vmfne.vv` (Vector Mask Set If Float Not Equal) instruction.
(decl rv_vmfne_vv (VReg VReg VecOpMasking VState) VReg)
(rule (rv_vmfne_vv vs2 vs1 mask vstate)
(vec_alu_rrr (VecAluOpRRR.VmfneVV) vs2 vs1 mask vstate))

;; Helper for emitting the `vmfne.vf` (Vector Mask Set If Float Not Equal) instruction.
(decl rv_vmfne_vf (VReg FReg VecOpMasking VState) VReg)
(rule (rv_vmfne_vf vs2 vs1 mask vstate)
(vec_alu_rrr (VecAluOpRRR.VmfneVF) vs2 vs1 mask vstate))

;; Helper for emitting the `vmflt.vv` (Vector Mask Set If Float Less Than) instruction.
(decl rv_vmflt_vv (VReg VReg VecOpMasking VState) VReg)
(rule (rv_vmflt_vv vs2 vs1 mask vstate)
(vec_alu_rrr (VecAluOpRRR.VmfltVV) vs2 vs1 mask vstate))

;; Helper for emitting the `vmflt.vf` (Vector Mask Set If Float Less Than) instruction.
(decl rv_vmflt_vf (VReg FReg VecOpMasking VState) VReg)
(rule (rv_vmflt_vf vs2 vs1 mask vstate)
(vec_alu_rrr (VecAluOpRRR.VmfltVF) vs2 vs1 mask vstate))

;; Helper for emitting the `vmfle.vv` (Vector Mask Set If Float Less Than Or Equal) instruction.
(decl rv_vmfle_vv (VReg VReg VecOpMasking VState) VReg)
(rule (rv_vmfle_vv vs2 vs1 mask vstate)
(vec_alu_rrr (VecAluOpRRR.VmfleVV) vs2 vs1 mask vstate))

;; Helper for emitting the `vmfle.vf` (Vector Mask Set If Float Less Than Or Equal) instruction.
(decl rv_vmfle_vf (VReg FReg VecOpMasking VState) VReg)
(rule (rv_vmfle_vf vs2 vs1 mask vstate)
(vec_alu_rrr (VecAluOpRRR.VmfleVF) vs2 vs1 mask vstate))

;; Helper for emitting the `vmfgt.vv` (Vector Mask Set If Float Greater Than) instruction.
;; This is an alias for `vmflt.vv` with the operands inverted.
(decl rv_vmfgt_vv (VReg VReg VecOpMasking VState) VReg)
(rule (rv_vmfgt_vv vs2 vs1 mask vstate) (rv_vmflt_vv vs1 vs2 mask vstate))

;; Helper for emitting the `vmfgt.vf` (Vector Mask Set If Float Greater Than) instruction.
(decl rv_vmfgt_vf (VReg FReg VecOpMasking VState) VReg)
(rule (rv_vmfgt_vf vs2 vs1 mask vstate)
(vec_alu_rrr (VecAluOpRRR.VmfgtVF) vs2 vs1 mask vstate))

;; Helper for emitting the `vmfge.vv` (Vector Mask Set If Float Greater Than Or Equal) instruction.
;; This is an alias for `vmfle.vv` with the operands inverted.
(decl rv_vmfge_vv (VReg VReg VecOpMasking VState) VReg)
(rule (rv_vmfge_vv vs2 vs1 mask vstate) (rv_vmfle_vv vs1 vs2 mask vstate))

;; Helper for emitting the `vmfge.vf` (Vector Mask Set If Float Greater Than Or Equal) instruction.
(decl rv_vmfge_vf (VReg FReg VecOpMasking VState) VReg)
(rule (rv_vmfge_vf vs2 vs1 mask vstate)
(vec_alu_rrr (VecAluOpRRR.VmfgeVF) vs2 vs1 mask vstate))

;; Helper for emitting the `vzext.vf2` instruction.
;; Zero-extend SEW/2 source to SEW destination
(decl rv_vzext_vf2 (VReg VecOpMasking VState) VReg)
Expand Down Expand Up @@ -1161,6 +1235,40 @@
(rule (rv_vnclipu_wi vs2 imm mask vstate)
(vec_alu_rr_uimm5 (VecAluOpRRImm5.VnclipuWI) vs2 imm mask vstate))

;; Helper for emitting the `vmand.mm` (Mask Bitwise AND) instruction.
;;
;; vd.mask[i] = vs2.mask[i] && vs1.mask[i]
(decl rv_vmand_mm (VReg VReg VState) VReg)
(rule (rv_vmand_mm vs2 vs1 vstate)
(vec_alu_rrr (VecAluOpRRR.VmandMM) vs2 vs1 (unmasked) vstate))

;; Helper for emitting the `vmor.mm` (Mask Bitwise OR) instruction.
;;
;; vd.mask[i] = vs2.mask[i] || vs1.mask[i]
(decl rv_vmor_mm (VReg VReg VState) VReg)
(rule (rv_vmor_mm vs2 vs1 vstate)
(vec_alu_rrr (VecAluOpRRR.VmorMM) vs2 vs1 (unmasked) vstate))

;; Helper for emitting the `vmnand.mm` (Mask Bitwise NAND) instruction.
;;
;; vd.mask[i] = !(vs2.mask[i] && vs1.mask[i])
(decl rv_vmnand_mm (VReg VReg VState) VReg)
(rule (rv_vmnand_mm vs2 vs1 vstate)
(vec_alu_rrr (VecAluOpRRR.VmnandMM) vs2 vs1 (unmasked) vstate))

;; Helper for emitting the `vmnot.m` (Mask Bitwise NOT) instruction.
;; This is an alias for `vmnand.mm vd, vs, vs`
;;
;; vd.mask[i] = !vs.mask[i]
(decl rv_vmnot_m (VReg VState) VReg)
(rule (rv_vmnot_m vs vstate) (rv_vmnand_mm vs vs vstate))

;; Helper for emitting the `vmnor.mm` (Mask Bitwise NOR) instruction.
;;
;; vd.mask[i] = !(vs2.mask[i] || vs1.mask[i])
(decl rv_vmnor_mm (VReg VReg VState) VReg)
(rule (rv_vmnor_mm vs2 vs1 vstate)
(vec_alu_rrr (VecAluOpRRR.VmnorMM) vs2 vs1 (unmasked) vstate))

;;;; Multi-Instruction Helpers ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

Expand Down Expand Up @@ -1378,3 +1486,127 @@

(rule 4 (gen_icmp_mask (ty_vec_fits_in_register ty) (IntCC.SignedGreaterThanOrEqual) (replicated_imm5 x) y)
(rv_vmsle_vi y x (unmasked) ty))



;; Builds a vector mask corresponding to the FloatCC operation.
(decl gen_fcmp_mask (Type FloatCC Value Value) VReg)

;; FloatCC.Equal

(rule 0 (gen_fcmp_mask (ty_vec_fits_in_register ty) (FloatCC.Equal) x y)
(rv_vmfeq_vv x y (unmasked) ty))

(rule 1 (gen_fcmp_mask (ty_vec_fits_in_register ty) (FloatCC.Equal) x (splat y))
(rv_vmfeq_vf x y (unmasked) ty))

(rule 2 (gen_fcmp_mask (ty_vec_fits_in_register ty) (FloatCC.Equal) (splat x) y)
(rv_vmfeq_vf y x (unmasked) ty))

;; FloatCC.NotEqual
;; Note: This is UnorderedNotEqual. It is the only unoredered comparison that is not named as such.

(rule 0 (gen_fcmp_mask (ty_vec_fits_in_register ty) (FloatCC.NotEqual) x y)
(rv_vmfne_vv x y (unmasked) ty))

(rule 1 (gen_fcmp_mask (ty_vec_fits_in_register ty) (FloatCC.NotEqual) x (splat y))
(rv_vmfne_vf x y (unmasked) ty))

(rule 2 (gen_fcmp_mask (ty_vec_fits_in_register ty) (FloatCC.NotEqual) (splat x) y)
(rv_vmfne_vf y x (unmasked) ty))

;; FloatCC.LessThan

(rule 0 (gen_fcmp_mask (ty_vec_fits_in_register ty) (FloatCC.LessThan) x y)
(rv_vmflt_vv x y (unmasked) ty))

(rule 1 (gen_fcmp_mask (ty_vec_fits_in_register ty) (FloatCC.LessThan) x (splat y))
(rv_vmflt_vf x y (unmasked) ty))

(rule 2 (gen_fcmp_mask (ty_vec_fits_in_register ty) (FloatCC.LessThan) (splat x) y)
(rv_vmfgt_vf y x (unmasked) ty))

;; FloatCC.LessThanOrEqual

(rule 0 (gen_fcmp_mask (ty_vec_fits_in_register ty) (FloatCC.LessThanOrEqual) x y)
(rv_vmfle_vv x y (unmasked) ty))

(rule 1 (gen_fcmp_mask (ty_vec_fits_in_register ty) (FloatCC.LessThanOrEqual) x (splat y))
(rv_vmfle_vf x y (unmasked) ty))

(rule 2 (gen_fcmp_mask (ty_vec_fits_in_register ty) (FloatCC.LessThanOrEqual) (splat x) y)
(rv_vmfge_vf y x (unmasked) ty))

;; FloatCC.GreaterThan

(rule 0 (gen_fcmp_mask (ty_vec_fits_in_register ty) (FloatCC.GreaterThan) x y)
(rv_vmfgt_vv x y (unmasked) ty))

(rule 1 (gen_fcmp_mask (ty_vec_fits_in_register ty) (FloatCC.GreaterThan) x (splat y))
(rv_vmfgt_vf x y (unmasked) ty))

(rule 2 (gen_fcmp_mask (ty_vec_fits_in_register ty) (FloatCC.GreaterThan) (splat x) y)
(rv_vmflt_vf y x (unmasked) ty))

;; FloatCC.GreaterThanOrEqual

(rule 0 (gen_fcmp_mask (ty_vec_fits_in_register ty) (FloatCC.GreaterThanOrEqual) x y)
(rv_vmfge_vv x y (unmasked) ty))

(rule 1 (gen_fcmp_mask (ty_vec_fits_in_register ty) (FloatCC.GreaterThanOrEqual) x (splat y))
(rv_vmfge_vf x y (unmasked) ty))

(rule 2 (gen_fcmp_mask (ty_vec_fits_in_register ty) (FloatCC.GreaterThanOrEqual) (splat x) y)
(rv_vmfle_vf y x (unmasked) ty))

;; FloatCC.Ordered

(rule 0 (gen_fcmp_mask (ty_vec_fits_in_register ty) (FloatCC.Ordered) x y)
(rv_vmand_mm
(gen_fcmp_mask ty (FloatCC.Equal) x x)
(gen_fcmp_mask ty (FloatCC.Equal) y y)
ty))

;; FloatCC.Unordered

(rule 0 (gen_fcmp_mask (ty_vec_fits_in_register ty) (FloatCC.Unordered) x y)
(rv_vmor_mm
(gen_fcmp_mask ty (FloatCC.NotEqual) x x)
(gen_fcmp_mask ty (FloatCC.NotEqual) y y)
ty))

;; FloatCC.OrderedNotEqual

(rule 0 (gen_fcmp_mask (ty_vec_fits_in_register ty) (FloatCC.OrderedNotEqual) x y)
(rv_vmor_mm
(gen_fcmp_mask ty (FloatCC.LessThan) x y)
(gen_fcmp_mask ty (FloatCC.LessThan) y x)
ty))

;; FloatCC.UnorderedOrEqual

(rule 0 (gen_fcmp_mask (ty_vec_fits_in_register ty) (FloatCC.UnorderedOrEqual) x y)
(rv_vmnor_mm
(gen_fcmp_mask ty (FloatCC.LessThan) x y)
(gen_fcmp_mask ty (FloatCC.LessThan) y x)
ty))

;; FloatCC.UnorderedOrGreaterThan

(rule 0 (gen_fcmp_mask (ty_vec_fits_in_register ty) (FloatCC.UnorderedOrGreaterThan) x y)
(rv_vmnot_m (gen_fcmp_mask ty (FloatCC.LessThanOrEqual) x y) ty))

;; FloatCC.UnorderedOrGreaterThanOrEqual

(rule 0 (gen_fcmp_mask (ty_vec_fits_in_register ty) (FloatCC.UnorderedOrGreaterThanOrEqual) x y)
(rv_vmnot_m (gen_fcmp_mask ty (FloatCC.LessThan) x y) ty))

;; FloatCC.UnorderedOrLessThan

(rule 0 (gen_fcmp_mask (ty_vec_fits_in_register ty) (FloatCC.UnorderedOrLessThan) x y)
(rv_vmnot_m (gen_fcmp_mask ty (FloatCC.GreaterThanOrEqual) x y) ty))

;; FloatCC.UnorderedOrLessThanOrEqual

(rule 0 (gen_fcmp_mask (ty_vec_fits_in_register ty) (FloatCC.UnorderedOrLessThanOrEqual) x y)
(rv_vmnot_m (gen_fcmp_mask ty (FloatCC.GreaterThan) x y) ty))
6 changes: 4 additions & 2 deletions cranelift/codegen/src/isa/riscv64/lower.isle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1462,10 +1462,12 @@


;;;;; Rules for `fcmp`;;;;;;;;;
(rule
(lower (fcmp cc x @ (value_type ty) y))
(rule 0 (lower (fcmp cc x @ (value_type (ty_scalar_float ty)) y))
(cmp_value (emit_fcmp cc ty x y)))

(rule 1 (lower (fcmp cc x @ (value_type (ty_vec_fits_in_register ty)) y))
(gen_expand_mask ty (gen_fcmp_mask ty cc x y)))

;;;;; Rules for `func_addr`;;;;;;;;;
(rule
(lower (func_addr (func_ref_data _ name _)))
Expand Down
Loading

0 comments on commit 3cab644

Please sign in to comment.