// Copyright 2025 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package main import ( "bytes" "fmt" "simd/archsimd/_gen/sgutil" "slices" "text/template" ) // Helper type to make template map initialization less repetitive // (and also remove a chance for errors.) type intrinsicTemplateMap struct { sgutil.InsertMap[string, *template.Template] } func templateNamed(name string, templ string) *template.Template { // Append end of line templ += "\n" t := template.New(name) archInfo := CurrentArch() sysArch := "sys." + archInfo.ArchUpper t.Funcs(template.FuncMap{ "GetSysArch": func() string { return sysArch }, "GetArchUpper": func() string { return archInfo.ArchUpper }, "Hasmask": func() bool { return archInfo.Arch == "amd64" }, }) return template.Must(t.Parse(templ)) } // Add creates a template named "name" after appending "\n" to the // template, and returns the input so that additions may be chained. // This helps make template initialization easy to order and easy to read. func (rtm *intrinsicTemplateMap) Add(name string, templ string) *intrinsicTemplateMap { rtm.InsertMap.Put(name, templateNamed(name, templ)) return rtm } // writeSIMDIntrinsics generates the intrinsic mappings and writes it to simdintrinsics.go // within the specified directory. func writeSIMDIntrinsics(ops []Operation, typeMap simdTypeMap) *bytes.Buffer { // These are defined here to avoid init-order problems with GetSysArch GetArchUpper etc which depend on flag values var header = templateNamed("header", `package ssagen import ( "cmd/compile/internal/ir" "cmd/compile/internal/ssa" "cmd/compile/internal/types" "cmd/internal/sys" ) func simd{{GetArchUpper}}Intrinsics(addF func(pkg, fn string, b intrinsicBuilder, archFamilies ...sys.ArchFamily)) { `) var intrinsicTemplates = new(intrinsicTemplateMap). Add("op1", ` addF(simdPackage, "{{(index .In 0).Go}}.{{.Go}}", opLen1(ssa.Op{{.GenericName}}, {{.SSAType}}), {{GetSysArch}})`). Add("op2", ` addF(simdPackage, "{{(index .In 0).Go}}.{{.Go}}", opLen2(ssa.Op{{.GenericName}}, {{.SSAType}}), {{GetSysArch}})`). Add("op2_21", ` addF(simdPackage, "{{(index .In 0).Go}}.{{.Go}}", opLen2_21(ssa.Op{{.GenericName}}, {{.SSAType}}), {{GetSysArch}})`). Add("op2_21Type1", `addF(simdPackage, "{{(index .In 1).Go}}.{{.Go}}", opLen2_21(ssa.Op{{.GenericName}}, {{.SSAType}}), {{GetSysArch}})`). Add("op3", ` addF(simdPackage, "{{(index .In 0).Go}}.{{.Go}}", opLen3(ssa.Op{{.GenericName}}, {{.SSAType}}), {{GetSysArch}})`). Add("op3_21", ` addF(simdPackage, "{{(index .In 0).Go}}.{{.Go}}", opLen3_21(ssa.Op{{.GenericName}}, {{.SSAType}}), {{GetSysArch}})`). Add("op3_21Type1", `addF(simdPackage, "{{(index .In 1).Go}}.{{.Go}}", opLen3_21(ssa.Op{{.GenericName}}, {{.SSAType}}), {{GetSysArch}})`). Add("op3_231Type1", `addF(simdPackage, "{{(index .In 1).Go}}.{{.Go}}", opLen3_231(ssa.Op{{.GenericName}}, {{.SSAType}}), {{GetSysArch}})`). Add("op3_31Zero3", `addF(simdPackage, "{{(index .In 2).Go}}.{{.Go}}", opLen3_31Zero3(ssa.Op{{.GenericName}}, {{.SSAType}}), {{GetSysArch}})`). Add("op4", ` addF(simdPackage, "{{(index .In 0).Go}}.{{.Go}}", opLen4(ssa.Op{{.GenericName}}, {{.SSAType}}), {{GetSysArch}})`). Add("op4_231Type1", `addF(simdPackage, "{{(index .In 1).Go}}.{{.Go}}", opLen4_231(ssa.Op{{.GenericName}}, {{.SSAType}}), {{GetSysArch}})`). Add("op4_31", ` addF(simdPackage, "{{(index .In 2).Go}}.{{.Go}}", opLen4_31(ssa.Op{{.GenericName}}, {{.SSAType}}), {{GetSysArch}})`). Add("op1Imm", ` addF(simdPackage, "{{(index .In 1).Go}}.{{.Go}}", opLen1Imm(ssa.Op{{.GenericName}}, {{.SSAType}}, {{(index .In 0).ImmOffset}}, {{(index .In 0).ImmMax}}), {{GetSysArch}})`). Add("op1Imm8", ` addF(simdPackage, "{{(index .In 1).Go}}.{{.Go}}", opLen1Imm8(ssa.Op{{.GenericName}}, {{.SSAType}}, {{(index .In 0).ImmOffset}}), {{GetSysArch}})`). Add("op2Imm", ` addF(simdPackage, "{{(index .In 1).Go}}.{{.Go}}", opLen2Imm(ssa.Op{{.GenericName}}, {{.SSAType}}, {{(index .In 0).ImmOffset}}, {{(index .In 0).ImmMax}}), {{GetSysArch}})`). Add("op2Imm8", ` addF(simdPackage, "{{(index .In 1).Go}}.{{.Go}}", opLen2Imm8(ssa.Op{{.GenericName}}, {{.SSAType}}, {{(index .In 0).ImmOffset}}), {{GetSysArch}})`). Add("op2Imm8_2I", ` addF(simdPackage, "{{(index .In 1).Go}}.{{.Go}}", opLen2Imm8_2I(ssa.Op{{.GenericName}}, {{.SSAType}}, {{(index .In 0).ImmOffset}}), {{GetSysArch}})`). Add("op2Imm_2I", ` addF(simdPackage, "{{(index .In 1).Go}}.{{.Go}}", opLen2Imm_2I(ssa.Op{{.GenericName}}, {{.SSAType}}, {{(index .In 0).ImmOffset}}, {{(index .In 0).ImmMax}}), {{GetSysArch}})`). Add("op2Imm8_II", ` addF(simdPackage, "{{(index .In 1).Go}}.{{.Go}}", opLen2Imm8_II(ssa.Op{{.GenericName}}, {{.SSAType}}, {{(index .In 0).ImmOffset}}), {{GetSysArch}})`). Add("op2Imm8_SHA1RNDS4", `addF(simdPackage, "{{(index .In 1).Go}}.{{.Go}}", opLen2Imm8_SHA1RNDS4(ssa.Op{{.GenericName}}, {{.SSAType}}, {{(index .In 0).ImmOffset}}), {{GetSysArch}})`). Add("op2ImmVecAsScalar", `addF(simdPackage, "{{(index .In 2).Go}}.{{.Go}}", opLen2Imm(ssa.Op{{.GenericName}}, {{.SSAType}}, {{(index .In 0).ImmOffset}}, {{(index .In 0).ImmMax}}), {{GetSysArch}})`). Add("op3Imm8", ` addF(simdPackage, "{{(index .In 1).Go}}.{{.Go}}", opLen3Imm8(ssa.Op{{.GenericName}}, {{.SSAType}}, {{(index .In 0).ImmOffset}}), {{GetSysArch}})`). Add("op3Imm8_2I", ` addF(simdPackage, "{{(index .In 1).Go}}.{{.Go}}", opLen3Imm8_2I(ssa.Op{{.GenericName}}, {{.SSAType}}, {{(index .In 0).ImmOffset}}), {{GetSysArch}})`). Add("op4Imm8", ` addF(simdPackage, "{{(index .In 1).Go}}.{{.Go}}", opLen4Imm8(ssa.Op{{.GenericName}}, {{.SSAType}}, {{(index .In 0).ImmOffset}}), {{GetSysArch}})`) var loadStore = templateNamed("loadStore", ` addF(simdPackage, "Load{{.Name}}Array", simdLoad(), {{GetSysArch}}) addF(simdPackage, "{{.Name}}.StoreArray", simdStore(), {{GetSysArch}})`) var mask = templateNamed("mask", ` addF(simdPackage, "{{.Name}}.To{{.VectorCounterpart}}", func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { return args[0] }, {{GetSysArch}}) addF(simdPackage, "{{.VectorCounterpart}}.asMask", func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { return args[0] }, {{GetSysArch}}) addF(simdPackage, "{{.Name}}.And", opLen2(ssa.OpAnd{{.ReshapedVectorWithAndOr}}, types.TypeVec{{.Size}}), {{GetSysArch}}) addF(simdPackage, "{{.Name}}.Or", opLen2(ssa.OpOr{{.ReshapedVectorWithAndOr}}, types.TypeVec{{.Size}}), {{GetSysArch}}) {{- if eq GetSysArch "sys.ARM64"}} addF(simdPackage, "{{.Name}}.Not", opLen1(ssa.OpNot{{.ReshapedVectorWithAndOr}}, types.TypeVec{{.Size}}), {{GetSysArch}}) {{- else}} addF(simdPackage, "{{.Name}}FromBits", simdCvtVToMask({{.ElemBits}}, {{.Lanes}}), {{GetSysArch}}) addF(simdPackage, "{{.Name}}.ToBits", simdCvtMaskToV({{.ElemBits}}, {{.Lanes}}), {{GetSysArch}}) {{- end}}`) var maskedLoadStore = templateNamed("maskedLoadStore", ` addF(simdPackage, "{{.Name}}.StoreArrayMasked", simdMaskedStore(ssa.OpStoreMasked{{.ElemBits}}), sys.AMD64)`) var vectorConversion = templateNamed("vectorConversion", ` addF(simdPackage, "{{.Tsrc.Name}}.As{{.Tdst.Name}}", func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { return args[0] }, {{GetSysArch}})`) var footer = `}` slices.SortFunc(ops, compareOperations) buffer := new(bytes.Buffer) buffer.WriteString(generatedHeader()) doTemplate := func(tpl *template.Template, data any) { if err := tpl.Execute(buffer, data); err != nil { panic(fmt.Errorf("failed to execute template %s: %w", tpl.Name(), err)) } } doTemplate(header, nil) doIntrinsic := func(name string, data any) { tpl := intrinsicTemplates.Get(name) if tpl == nil { panic(fmt.Errorf("template %s not found", name)) } doTemplate(tpl, data) } for _, op := range ops { if op.NoTypes != nil && *op.NoTypes == "true" { continue } if op.SkipMaskedMethod() { continue } // Cannot have an intrinsic w/o generics, at least for now. if op.NoGenericOps != nil && *op.NoGenericOps == "true" { continue } if s, op, err := classifyOp(op); err == nil { if s == "op2Imm" { idxVecAsScalar, err := checkVecAsScalar(op) if err != nil { panic(err) } if idxVecAsScalar >= 0 { s += "VecAsScalar" } } doIntrinsic(s, op) } else { panic(fmt.Errorf("failed to classify op %v: %w", op.Go, err)) } } var TypeDotMethodIntrinsicAMD64 = templateOf(`addF(simdPackage, "{{.TypeDotMethod}}", func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { return args[0] }, sys.AMD64) `, "amd64 type dot method intrinsics") var TypeDotMethodIntrinsicARM64 = templateOf(`addF(simdPackage, "{{.TypeDotMethod}}", func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { return args[0] }, sys.ARM64) `, "arm64 type dot method intrinsics") for _, conv := range vConvertFromTypeMap(typeMap) { // Old As intrinsic from, to := &conv.Tsrc, &conv.Tdst doTemplate(vectorConversion, conv) // New style factored conversion intrinsics always involve at least one unsigned type if from.Name[0] != 'U' && to.Name[0] != 'U' { continue } // Only emit the intrinsic if lanes are equal OR both are unsigned if from.Lanes != to.Lanes && (from.Name[0] != 'U' || to.Name[0] != 'U') { continue } var typeDotMethodIntrinsic *template.Template switch CurrentArch().Arch { case "amd64": typeDotMethodIntrinsic = TypeDotMethodIntrinsicAMD64 case "arm64": typeDotMethodIntrinsic = TypeDotMethodIntrinsicARM64 default: panic(fmt.Errorf("unsupported arch %q for type dot method intrinsics", CurrentArch().Arch)) } sgutil.Conversion(from, to).ExecuteIntrinsicTemplateOfTypeDotMethod(buffer, typeDotMethodIntrinsic) } for _, typ := range typesFromTypeMap(typeMap) { if typ.Type != "mask" { loadStore.Execute(buffer, typ) } } // Masked loads/stores are AVX2/AVX512 only (not available on ARM64 NEON). // TODO: Reconsider for ARM64 SVE which supports predicated loads/stores. if CurrentArch().Arch == "amd64" { for _, typ := range typesFromTypeMap(typeMap) { if typ.MaskedLoadStoreFilter() { doTemplate(maskedLoadStore, typ) } } } for _, m := range masksFromTypeMap(typeMap) { doTemplate(mask, m) } buffer.WriteString(footer) return buffer }