Source file src/simd/archsimd/_gen/simdgen/gen_simdTypes.go

     1  // Copyright 2025 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package main
     6  
     7  import (
     8  	"bytes"
     9  	"cmp"
    10  	"fmt"
    11  	"maps"
    12  	"slices"
    13  	"sort"
    14  	"strings"
    15  	"text/template"
    16  	"unicode"
    17  
    18  	"simd/archsimd/_gen/sgutil"
    19  )
    20  
    21  type simdType struct {
    22  	Name                    string // The go type name of this simd type, for example Int32x4.
    23  	Lanes                   int    // The number of elements in this vector/mask.
    24  	Base                    string // The element's type, like for Int32x4 it will be int32.
    25  	Fields                  string // The struct fields, it should be right formatted.
    26  	Type                    string // Either "mask" or "vreg"
    27  	VectorCounterpart       string // For mask use only: just replacing the "Mask" in [simdType.Name] with "Int"
    28  	ReshapedVectorWithAndOr string // For mask use only: vector AND and OR are only available in some shape with element width 32.
    29  	Size                    int    // The size of the vector type
    30  	HasNot                  bool   // True when this mask type supports Not()
    31  }
    32  
    33  func (x simdType) ElemBits() int {
    34  	return x.Size / x.Lanes
    35  }
    36  
    37  func (x *simdType) Name_() string {
    38  	return x.Name
    39  }
    40  
    41  func (x simdType) Article() string {
    42  	if strings.HasPrefix(x.Name, "Int") {
    43  		return "an"
    44  	}
    45  	return "a" // Float, Uint
    46  }
    47  
    48  // LanesContainer returns the smallest int/uint bit size that is
    49  // large enough to hold one bit for each lane.  E.g., Mask32x4
    50  // is 4 lanes, and a uint8 is the smallest uint that has 4 bits.
    51  func (x simdType) LanesContainer() int {
    52  	if x.Lanes > 64 {
    53  		panic("too many lanes")
    54  	}
    55  	if x.Lanes > 32 {
    56  		return 64
    57  	}
    58  	if x.Lanes > 16 {
    59  		return 32
    60  	}
    61  	if x.Lanes > 8 {
    62  		return 16
    63  	}
    64  	return 8
    65  }
    66  
    67  // MaskedLoadStoreFilter encodes which simd type type currently
    68  // get masked loads/stores generated, it is used in two places,
    69  // this forces coordination.
    70  func (x simdType) MaskedLoadStoreFilter() bool {
    71  	return x.Size == 512 || x.ElemBits() >= 32 && x.Type != "mask"
    72  }
    73  
    74  func (x simdType) IntelSizeSuffix() string {
    75  	switch x.ElemBits() {
    76  	case 8:
    77  		return "B"
    78  	case 16:
    79  		return "W"
    80  	case 32:
    81  		return "D"
    82  	case 64:
    83  		return "Q"
    84  	}
    85  	panic("oops")
    86  }
    87  
    88  func (x simdType) MaskedLoadDoc() string {
    89  	if x.Size == 512 || x.ElemBits() < 32 {
    90  		return fmt.Sprintf("// Asm: VMOVDQU%d.Z, CPU Feature: AVX512", x.ElemBits())
    91  	} else {
    92  		return fmt.Sprintf("// Asm: VMASKMOV%s, CPU Feature: AVX2", x.IntelSizeSuffix())
    93  	}
    94  }
    95  
    96  func (x simdType) MaskedStoreDoc() string {
    97  	if x.Size == 512 || x.ElemBits() < 32 {
    98  		return fmt.Sprintf("// Asm: VMOVDQU%d, CPU Feature: AVX512", x.ElemBits())
    99  	} else {
   100  		return fmt.Sprintf("// Asm: VMASKMOV%s, CPU Feature: AVX2", x.IntelSizeSuffix())
   101  	}
   102  }
   103  
   104  func (x simdType) ToBitsDoc() string {
   105  	if x.Size == 512 || x.ElemBits() == 16 {
   106  		return fmt.Sprintf("// Asm: KMOV%s, CPU Features: AVX512", x.IntelSizeSuffix())
   107  	}
   108  	// 128/256 bit vectors with 8, 32, 64 bit elements
   109  	var asm string
   110  	var feat string
   111  	switch x.ElemBits() {
   112  	case 8:
   113  		asm = "VPMOVMSKB"
   114  		if x.Size == 256 {
   115  			feat = "AVX2"
   116  		} else {
   117  			feat = "AVX"
   118  		}
   119  	case 32:
   120  		asm = "VMOVMSKPS"
   121  		feat = "AVX"
   122  	case 64:
   123  		asm = "VMOVMSKPD"
   124  		feat = "AVX"
   125  	default:
   126  		panic("unexpected ElemBits")
   127  	}
   128  	return fmt.Sprintf("// Asm: %s, CPU Features: %s", asm, feat)
   129  }
   130  
   131  func compareSimdTypes(x, y simdType) int {
   132  	// "vreg" then "mask"
   133  	if c := -compareNatural(x.Type, y.Type); c != 0 {
   134  		return c
   135  	}
   136  	// want "flo" < "int" < "uin" (and then 8 < 16 < 32 < 64),
   137  	// not "int16" < "int32" < "int64" < "int8")
   138  	// so limit comparison to first 3 bytes in string.
   139  	if c := compareNatural(x.Base[:3], y.Base[:3]); c != 0 {
   140  		return c
   141  	}
   142  	// base type size, 8 < 16 < 32 < 64
   143  	if c := x.ElemBits() - y.ElemBits(); c != 0 {
   144  		return c
   145  	}
   146  	// vector size last
   147  	return x.Size - y.Size
   148  }
   149  
   150  type simdTypeMap map[int][]simdType
   151  
   152  type simdTypePair struct {
   153  	Tsrc simdType
   154  	Tdst simdType
   155  }
   156  
   157  func compareSimdTypePairs(x, y simdTypePair) int {
   158  	c := compareSimdTypes(x.Tsrc, y.Tsrc)
   159  	if c != 0 {
   160  		return c
   161  	}
   162  	return compareSimdTypes(x.Tdst, y.Tdst)
   163  }
   164  
   165  func simdPackageHeader() string {
   166  	return generatedHeader() + `
   167  //go:build goexperiment.simd
   168  
   169  package archsimd
   170  `
   171  }
   172  
   173  const simdTypesTemplates = `
   174  {{define "sizeTmpl"}}
   175  // v{{.}} is a tag type that tells the compiler that this is really {{.}}-bit SIMD
   176  type v{{.}} struct {
   177  	_{{.}} [0]func() // uncomparable
   178  }
   179  {{end}}
   180  
   181  {{define "typeTmpl"}}
   182  {{- if eq .Type "mask"}}
   183  // {{.Name}} is a mask for a SIMD vector of {{.Lanes}} {{.ElemBits}}-bit elements.
   184  {{- else}}
   185  // {{.Name}} is a {{.Size}}-bit SIMD vector of {{.Lanes}} {{.Base}}s.
   186  {{- end}}
   187  type {{.Name}} struct {
   188  {{.Fields}}
   189  }
   190  
   191  {{end}}
   192  `
   193  
   194  const simdFeaturesTemplate = `
   195  import "internal/cpu"
   196  
   197  type X86Features struct {}
   198  
   199  var X86 X86Features
   200  
   201  {{range .}}
   202  {{$f := .}}
   203  {{- if eq .Feature "AVX512"}}
   204  // {{.Feature}} returns whether the CPU supports the AVX512F+CD+BW+DQ+VL features.
   205  //
   206  // These five CPU features are bundled together, and no use of AVX-512
   207  // is allowed unless all of these features are supported together.
   208  // Nearly every CPU that has shipped with any support for AVX-512 has
   209  // supported all five of these features.
   210  {{- else -}}
   211  // {{.Feature}} returns whether the CPU supports the {{.Feature}} feature.
   212  {{- end}}
   213  {{- if ne .ImpliesAll ""}}
   214  //
   215  // If it returns true, then the CPU also supports {{.ImpliesAll}}.
   216  {{- end}}
   217  //
   218  // {{.Feature}} is defined on all GOARCHes, but will only return true on
   219  // GOARCH {{.GoArch}}.
   220  func ({{.FeatureVar}}Features) {{.Feature}}() bool {
   221  {{- if .Virtual}}
   222  	return {{range $i, $dep := .Implies}}{{if $i}} && {{end}}cpu.{{$f.FeatureVar}}.Has{{$dep}}{{end}}
   223  {{- else}}
   224  	return cpu.{{.FeatureVar}}.Has{{.Feature}}
   225  {{- end}}
   226  }
   227  {{end}}
   228  `
   229  
   230  const simdLoadStoreTemplate = `
   231  // Len returns the number of elements in {{.Article}} {{.Name}}.
   232  func (x {{.Name}}) Len() int { return {{.Lanes}} }
   233  
   234  // Load{{.Name}}Array loads {{.Article}} {{.Name}} from an array.
   235  //
   236  //go:noescape
   237  func Load{{.Name}}Array(y *[{{.Lanes}}]{{.Base}}) {{.Name}}
   238  
   239  // StoreArray stores {{.Article}} {{.Name}} to an array.
   240  //
   241  //go:noescape
   242  func (x {{.Name}}) StoreArray(y *[{{.Lanes}}]{{.Base}})
   243  `
   244  
   245  const simdMaskFromValTemplate = `
   246  // {{.Name}}FromBits constructs a {{.Name}} from a bitmap value, where 1 means set for the indexed element, 0 means unset.
   247  {{- if ne .Lanes .LanesContainer}}
   248  // Only the lower {{.Lanes}} bits of y are used.
   249  {{- end}}
   250  //
   251  // Asm: KMOV{{.IntelSizeSuffix}}, CPU Feature: AVX512
   252  func {{.Name}}FromBits(y uint{{.LanesContainer}}) {{.Name}}
   253  
   254  // ToBits constructs a bitmap from a {{.Name}}, where 1 means set for the indexed element, 0 means unset.
   255  {{- if ne .Lanes .LanesContainer}}
   256  // Only the lower {{.Lanes}} bits of y are used.
   257  {{- end}}
   258  //
   259  {{.ToBitsDoc}}
   260  func (x {{.Name}}) ToBits() uint{{.LanesContainer}}
   261  `
   262  
   263  const simdMaskedLoadStoreTemplate = `
   264  // StoreArrayMasked stores {{.Article}} {{.Name}} to an array,
   265  // at those elements enabled by mask.
   266  //
   267  {{.MaskedStoreDoc}}
   268  //
   269  //go:noescape
   270  func (x {{.Name}}) StoreArrayMasked(y *[{{.Lanes}}]{{.Base}}, mask Mask{{.ElemBits}}x{{.Lanes}})
   271  `
   272  
   273  // Helper type to make template map initialization less repetitive
   274  // (and also remove a chance for errors.)
   275  type stubTemplateMap struct {
   276  	sgutil.InsertMap[string, *template.Template]
   277  }
   278  
   279  // Add creates a template named "name" after appending "\n" to the
   280  // template, and returns the input so that additions may be chained.
   281  // This helps make template initialization easy to order and easy to read.
   282  func (rtm *stubTemplateMap) Add(name string, templ string) *stubTemplateMap {
   283  	// Wrap in newlines.
   284  	templ = "\n" + templ + "\n"
   285  	ct := sgutil.TemplateNamed(name, templ)
   286  	rtm.InsertMap.Put(name, ct)
   287  	return rtm
   288  }
   289  
   290  var stubTemplates = new(stubTemplateMap)
   291  
   292  func init() {
   293  	st := stubTemplates
   294  
   295  	st.Add("op1", `{{if .Documentation}}{{.Documentation}}
   296  //{{end}}
   297  // Asm: {{.Asm}}, CPU Feature: {{.CPUFeature}}
   298  func ({{.Op0NameAndType "x"}}) {{.Go}}() {{.GoType}}`)
   299  
   300  	st.Add("op2", `{{if .Documentation}}{{.Documentation}}
   301  //{{end}}
   302  // Asm: {{.Asm}}, CPU Feature: {{.CPUFeature}}
   303  func ({{.Op0NameAndType "x"}}) {{.Go}}({{.Op1NameAndType "y"}}) {{.GoType}}`)
   304  
   305  	st.Add("op2_21", `{{if .Documentation}}{{.Documentation}}
   306  //{{end}}
   307  // Asm: {{.Asm}}, CPU Feature: {{.CPUFeature}}
   308  func ({{.Op1NameAndType "x"}}) {{.Go}}({{.Op0NameAndType "y"}}) {{.GoType}}`)
   309  
   310  	st.Add("op2_21Type1", `{{if .Documentation}}{{.Documentation}}
   311  //{{end}}
   312  // Asm: {{.Asm}}, CPU Feature: {{.CPUFeature}}
   313  func ({{.Op1NameAndType "x"}}) {{.Go}}({{.Op0NameAndType "y"}}) {{.GoType}}`)
   314  
   315  	st.Add("op3", `{{if .Documentation}}{{.Documentation}}
   316  //{{end}}
   317  // Asm: {{.Asm}}, CPU Feature: {{.CPUFeature}}
   318  func ({{.Op0NameAndType "x"}}) {{.Go}}({{.Op1NameAndType "y"}}, {{.Op2NameAndType "z"}}) {{.GoType}}`)
   319  
   320  	st.Add("op3_31Zero3", `{{if .Documentation}}{{.Documentation}}
   321  //{{end}}
   322  // Asm: {{.Asm}}, CPU Feature: {{.CPUFeature}}
   323  func ({{.Op2NameAndType "x"}}) {{.Go}}({{.Op1NameAndType "y"}}) {{.GoType}}`)
   324  
   325  	st.Add("op3_21", `{{if .Documentation}}{{.Documentation}}
   326  //{{end}}
   327  // Asm: {{.Asm}}, CPU Feature: {{.CPUFeature}}
   328  func ({{.Op1NameAndType "x"}}) {{.Go}}({{.Op0NameAndType "y"}}, {{.Op2NameAndType "z"}}) {{.GoType}}`)
   329  
   330  	st.Add("op3_21Type1", `{{if .Documentation}}{{.Documentation}}
   331  //{{end}}
   332  // Asm: {{.Asm}}, CPU Feature: {{.CPUFeature}}
   333  func ({{.Op1NameAndType "x"}}) {{.Go}}({{.Op0NameAndType "y"}}, {{.Op2NameAndType "z"}}) {{.GoType}}`)
   334  
   335  	st.Add("op3_231Type1", `{{if .Documentation}}{{.Documentation}}
   336  //{{end}}
   337  // Asm: {{.Asm}}, CPU Feature: {{.CPUFeature}}
   338  func ({{.Op1NameAndType "x"}}) {{.Go}}({{.Op2NameAndType "y"}}, {{.Op0NameAndType "z"}}) {{.GoType}}`)
   339  
   340  	st.Add("op2VecAsScalar", `{{if .Documentation}}{{.Documentation}}
   341  //{{end}}
   342  // Asm: {{.Asm}}, CPU Feature: {{.CPUFeature}}
   343  func ({{.Op0NameAndType "x"}}) {{.Go}}({{.Op1Name "y"}} uint{{(index .In 1).TreatLikeAScalarOfSize}}) {{(index .Out 0).Go}}`)
   344  
   345  	st.Add("op2ImmVecAsScalar", `{{if .Documentation}}{{.Documentation}}
   346  //{{end}}
   347  // {{.ImmName}} results in better performance when it's a constant, a non-constant value will be translated into a jump table.
   348  // Asm: {{.Asm}}, CPU Feature: {{.CPUFeature}}
   349  func ({{.Op2NameAndType "x"}}) {{.Go}}({{.ImmName}} {{.ImmType}}, v float{{(index .In 3).ElemBits}}) {{(index .Out 0).Go}}`)
   350  
   351  	st.Add("op3VecAsScalar", `{{if .Documentation}}{{.Documentation}}
   352  //{{end}}
   353  // Asm: {{.Asm}}, CPU Feature: {{.CPUFeature}}
   354  func ({{.Op0NameAndType "x"}}) {{.Go}}({{.Op1Name "y"}} uint{{(index .In 1).TreatLikeAScalarOfSize}}, {{.Op2NameAndType "z"}}) {{(index .Out 0).Go}}`)
   355  
   356  	st.Add("op4", `{{if .Documentation}}{{.Documentation}}
   357  //{{end}}
   358  // Asm: {{.Asm}}, CPU Feature: {{.CPUFeature}}
   359  func ({{.Op0NameAndType "x"}}) {{.Go}}({{.Op1NameAndType "y"}}, {{.Op2NameAndType "z"}}, {{.Op3NameAndType "u"}}) {{.GoType}}`)
   360  
   361  	st.Add("op4_231Type1", `{{if .Documentation}}{{.Documentation}}
   362  //{{end}}
   363  // Asm: {{.Asm}}, CPU Feature: {{.CPUFeature}}
   364  func ({{.Op1NameAndType "x"}}) {{.Go}}({{.Op2NameAndType "y"}}, {{.Op0NameAndType "z"}}, {{.Op3NameAndType "u"}}) {{.GoType}}`)
   365  
   366  	st.Add("op4_31", `{{if .Documentation}}{{.Documentation}}
   367  //{{end}}
   368  // Asm: {{.Asm}}, CPU Feature: {{.CPUFeature}}
   369  func ({{.Op2NameAndType "x"}}) {{.Go}}({{.Op1NameAndType "y"}}, {{.Op0NameAndType "z"}}, {{.Op3NameAndType "u"}}) {{.GoType}}`)
   370  
   371  	st.Add("op1Imm", `{{if .Documentation}}{{.Documentation}}
   372  //{{end}}
   373  // A non-constant value of {{.ImmName}} may result in significantly worse performance for this operation.
   374  //
   375  // Asm: {{.Asm}}, CPU Feature: {{.CPUFeature}}
   376  func ({{.Op1NameAndType "x"}}) {{.Go}}({{.ImmName}} {{.ImmType}}) {{.GoType}}`)
   377  
   378  	st.Add("op1Imm8", `{{if .Documentation}}{{.Documentation}}
   379  //{{end}}
   380  // A non-constant value of {{.ImmName}} may result in significantly worse performance for this operation.
   381  //
   382  // Asm: {{.Asm}}, CPU Feature: {{.CPUFeature}}
   383  func ({{.Op1NameAndType "x"}}) {{.Go}}({{.ImmName}} {{.ImmType}}) {{.GoType}}`)
   384  
   385  	st.Add("op2Imm", `{{if .Documentation}}{{.Documentation}}
   386  //{{end}}
   387  // A non-constant value of {{.ImmName}} may result in significantly worse performance for this operation.
   388  //
   389  // Asm: {{.Asm}}, CPU Feature: {{.CPUFeature}}
   390  func ({{.Op1NameAndType "x"}}) {{.Go}}({{.ImmName}} {{.ImmType}}, {{.Op2NameAndType "y"}}) {{.GoType}}`)
   391  
   392  	st.Add("op2Imm8", `{{if .Documentation}}{{.Documentation}}
   393  //{{end}}
   394  // A non-constant value of {{.ImmName}} may result in significantly worse performance for this operation.
   395  //
   396  // Asm: {{.Asm}}, CPU Feature: {{.CPUFeature}}
   397  func ({{.Op1NameAndType "x"}}) {{.Go}}({{.ImmName}} {{.ImmType}}, {{.Op2NameAndType "y"}}) {{.GoType}}`)
   398  
   399  	// Special case for the instruction (in some versions at least) takes an immediate but treat it as a regular operand
   400  	st.Add("op1Imm8_rotate", `{{if .Documentation}}{{.Documentation}}
   401  //{{end}}
   402  // Emulated
   403  func ({{.Op1NameAndType "x"}}) {{.Go}}(dist uint64) {{.GoType}}`)
   404  
   405  	st.Add("op2Imm8_2I", `{{if .Documentation}}{{.Documentation}}
   406  //{{end}}
   407  // A non-constant value of {{.ImmName}} may result in significantly worse performance for this operation.
   408  //
   409  // Asm: {{.Asm}}, CPU Feature: {{.CPUFeature}}
   410  func ({{.Op1NameAndType "x"}}) {{.Go}}({{.Op2NameAndType "y"}}, {{.ImmName}} {{.ImmType}}) {{.GoType}}`)
   411  
   412  	st.Add("op2Imm_2I", `{{if .Documentation}}{{.Documentation}}
   413  //{{end}}
   414  // A non-constant value of {{.ImmName}} may result in significantly worse performance for this operation.
   415  //
   416  // Asm: {{.Asm}}, CPU Feature: {{.CPUFeature}}
   417  func ({{.Op1NameAndType "x"}}) {{.Go}}({{.Op2NameAndType "y"}}, {{.ImmName}} {{.ImmType}}) {{.GoType}}`)
   418  
   419  	st.Add("op2Imm8_II", `{{if .Documentation}}{{.Documentation}}
   420  //{{end}}
   421  // {{.ImmName}} should be between 0 and 3, inclusive; other values may result in a runtime panic.
   422  //
   423  // A non-constant value of {{.ImmName}} may result in significantly worse performance for this operation.
   424  //
   425  // Asm: {{.Asm}}, CPU Feature: {{.CPUFeature}}
   426  func ({{.Op1NameAndType "x"}}) {{.Go}}({{.ImmName}} {{.ImmType}}, {{.Op2NameAndType "y"}}) {{.GoType}}`)
   427  
   428  	st.Add("op2Imm8_SHA1RNDS4", `{{if .Documentation}}{{.Documentation}}
   429  //{{end}}
   430  // A non-constant value of {{.ImmName}} may result in significantly worse performance for this operation.
   431  //
   432  // Asm: {{.Asm}}, CPU Feature: {{.CPUFeature}}
   433  func ({{.Op1NameAndType "x"}}) {{.Go}}({{.ImmName}} {{.ImmType}}, {{.Op2NameAndType "y"}}) {{.GoType}}`)
   434  
   435  	st.Add("op3Imm8", `{{if .Documentation}}{{.Documentation}}
   436  //{{end}}
   437  // A non-constant value of {{.ImmName}} may result in significantly worse performance for this operation.
   438  //
   439  // Asm: {{.Asm}}, CPU Feature: {{.CPUFeature}}
   440  func ({{.Op1NameAndType "x"}}) {{.Go}}({{.ImmName}} {{.ImmType}}, {{.Op2NameAndType "y"}}, {{.Op3NameAndType "z"}}) {{.GoType}}`)
   441  
   442  	st.Add("op3Imm8_2I", `{{if .Documentation}}{{.Documentation}}
   443  //{{end}}
   444  // A non-constant value of {{.ImmName}} may result in significantly worse performance for this operation.
   445  //
   446  // Asm: {{.Asm}}, CPU Feature: {{.CPUFeature}}
   447  func ({{.Op1NameAndType "x"}}) {{.Go}}({{.Op2NameAndType "y"}}, {{.ImmName}} {{.ImmType}}, {{.Op3NameAndType "z"}}) {{.GoType}}`)
   448  
   449  	st.Add("op4Imm8", `{{if .Documentation}}{{.Documentation}}
   450  //{{end}}
   451  // A non-constant value of {{.ImmName}} may result in significantly worse performance for this operation.
   452  //
   453  // Asm: {{.Asm}}, CPU Feature: {{.CPUFeature}}
   454  func ({{.Op1NameAndType "x"}}) {{.Go}}({{.ImmName}} {{.ImmType}}, {{.Op2NameAndType "y"}}, {{.Op3NameAndType "z"}}, {{.Op4NameAndType "u"}}) {{.GoType}}`)
   455  
   456  	st.Add("mask", `// To{{.VectorCounterpart}} converts from {{.Name}} to {{.VectorCounterpart}}.
   457  // If element i in the mask is "true", all bits in element i of the resulting
   458  // vector will be set.
   459  func (from {{.Name}}) To{{.VectorCounterpart}}() (to {{.VectorCounterpart}})
   460  
   461  // asMask converts from {{.VectorCounterpart}} to {{.Name}}.
   462  func (from {{.VectorCounterpart}}) asMask() (to {{.Name}})
   463  
   464  func (x {{.Name}}) And(y {{.Name}}) {{.Name}}
   465  
   466  func (x {{.Name}}) Or(y {{.Name}}) {{.Name}}
   467  {{if .HasNot}}
   468  func (x {{.Name}}) Not() {{.Name}}
   469  {{end}}`)
   470  
   471  }
   472  
   473  // parseSIMDTypes groups go simd types by their vector sizes, and
   474  // returns a map whose key is the vector size, value is the simd type.
   475  func parseSIMDTypes(ops []Operation) simdTypeMap {
   476  	// TODO: maybe instead of going over ops, let's try go over types.yaml.
   477  	ret := map[int][]simdType{}
   478  	seen := map[string]struct{}{}
   479  	processArg := func(arg Operand) {
   480  		if arg.Class == "immediate" || arg.Class == "greg" {
   481  			// Immediates are not encoded as vector types.
   482  			return
   483  		}
   484  		if _, ok := seen[*arg.Go]; ok {
   485  			return
   486  		}
   487  		seen[*arg.Go] = struct{}{}
   488  
   489  		lanes := *arg.Lanes
   490  		base := fmt.Sprintf("%s%d", *arg.Base, *arg.ElemBits)
   491  		tagFieldNameS := fmt.Sprintf("%sx%d", base, lanes)
   492  		tagFieldS := fmt.Sprintf("%s v%d", tagFieldNameS, *arg.Bits)
   493  		valFieldS := fmt.Sprintf("vals%s[%d]%s", strings.Repeat(" ", len(tagFieldNameS)-3), lanes, base)
   494  		fields := fmt.Sprintf("\t%s\n\t%s", tagFieldS, valFieldS)
   495  		hasNot := CurrentArch().Arch == "arm64"
   496  		if arg.Class == "mask" {
   497  			vectorCounterpart := strings.ReplaceAll(*arg.Go, "Mask", "Int")
   498  			reshapedVectorWithAndOr := fmt.Sprintf("Int32x%d", *arg.Bits/32)
   499  			ret[*arg.Bits] = append(ret[*arg.Bits], simdType{*arg.Go, lanes, base, fields, arg.Class, vectorCounterpart, reshapedVectorWithAndOr, *arg.Bits, hasNot})
   500  			// In case the vector counterpart of a mask is not present, put its vector counterpart typedef into the map as well.
   501  			if _, ok := seen[vectorCounterpart]; !ok {
   502  				seen[vectorCounterpart] = struct{}{}
   503  				ret[*arg.Bits] = append(ret[*arg.Bits], simdType{vectorCounterpart, lanes, base, fields, "vreg", "", "", *arg.Bits, hasNot})
   504  			}
   505  		} else {
   506  			ret[*arg.Bits] = append(ret[*arg.Bits], simdType{*arg.Go, lanes, base, fields, arg.Class, "", "", *arg.Bits, hasNot})
   507  		}
   508  	}
   509  	for _, op := range ops {
   510  		for _, arg := range op.In {
   511  			processArg(arg)
   512  		}
   513  		for _, arg := range op.Out {
   514  			processArg(arg)
   515  		}
   516  	}
   517  	for _, v := range ret {
   518  		slices.SortFunc(v, compareSimdTypes)
   519  	}
   520  
   521  	return ret
   522  }
   523  
   524  func vConvertFromTypeMap(typeMap simdTypeMap) []simdTypePair {
   525  	v := []simdTypePair{}
   526  	for _, ts := range typeMap {
   527  		for i, tsrc := range ts {
   528  			for j, tdst := range ts {
   529  				if i != j && tsrc.Type == tdst.Type && tsrc.Type == "vreg" &&
   530  					tsrc.Lanes > 1 && tdst.Lanes > 1 {
   531  					v = append(v, simdTypePair{tsrc, tdst})
   532  				}
   533  			}
   534  		}
   535  	}
   536  	slices.SortFunc(v, compareSimdTypePairs)
   537  	return v
   538  }
   539  
   540  func masksFromTypeMap(typeMap simdTypeMap) []simdType {
   541  	m := []simdType{}
   542  	for _, ts := range typeMap {
   543  		for _, tsrc := range ts {
   544  			if tsrc.Type == "mask" {
   545  				m = append(m, tsrc)
   546  			}
   547  		}
   548  	}
   549  	slices.SortFunc(m, compareSimdTypes)
   550  	return m
   551  }
   552  
   553  func typesFromTypeMap(typeMap simdTypeMap) []simdType {
   554  	m := []simdType{}
   555  	for _, ts := range typeMap {
   556  		for _, tsrc := range ts {
   557  			if tsrc.Lanes > 1 {
   558  				m = append(m, tsrc)
   559  			}
   560  		}
   561  	}
   562  	slices.SortFunc(m, compareSimdTypes)
   563  	return m
   564  }
   565  
   566  // writeSIMDTypes generates the simd vector types into a bytes.Buffer
   567  func writeSIMDTypes(typeMap simdTypeMap) *bytes.Buffer {
   568  	t := templateOf(simdTypesTemplates, "types_amd64")
   569  	loadStore := templateOf(simdLoadStoreTemplate, "loadstore_amd64")
   570  	maskedLoadStore := templateOf(simdMaskedLoadStoreTemplate, "maskedloadstore_amd64")
   571  	maskFromVal := templateOf(simdMaskFromValTemplate, "maskFromVal_amd64")
   572  
   573  	buffer := new(bytes.Buffer)
   574  	buffer.WriteString(simdPackageHeader())
   575  
   576  	sizes := make([]int, 0, len(typeMap))
   577  	for size, types := range typeMap {
   578  		slices.SortFunc(types, compareSimdTypes)
   579  		sizes = append(sizes, size)
   580  	}
   581  	sort.Ints(sizes)
   582  
   583  	for _, size := range sizes {
   584  		if size <= 64 {
   585  			// these are scalar
   586  			continue
   587  		}
   588  		if err := t.ExecuteTemplate(buffer, "sizeTmpl", size); err != nil {
   589  			panic(fmt.Errorf("failed to execute size template for size %d: %w", size, err))
   590  		}
   591  		for _, typeDef := range typeMap[size] {
   592  			if typeDef.Lanes == 1 {
   593  				continue
   594  			}
   595  			if err := t.ExecuteTemplate(buffer, "typeTmpl", typeDef); err != nil {
   596  				panic(fmt.Errorf("failed to execute type template for type %s: %w", typeDef.Name, err))
   597  			}
   598  			if typeDef.Type != "mask" {
   599  				if err := loadStore.ExecuteTemplate(buffer, "loadstore_amd64", typeDef); err != nil {
   600  					panic(fmt.Errorf("failed to execute loadstore template for type %s: %w", typeDef.Name, err))
   601  				}
   602  				// restrict to AVX2 masked loads/stores first.
   603  				if CurrentArch().Arch == "amd64" && typeDef.MaskedLoadStoreFilter() {
   604  					if err := maskedLoadStore.ExecuteTemplate(buffer, "maskedloadstore_amd64", typeDef); err != nil {
   605  						panic(fmt.Errorf("failed to execute maskedloadstore template for type %s: %w", typeDef.Name, err))
   606  					}
   607  				}
   608  			} else {
   609  				// ARM64 NEON comparisons produce all-0/all-1 per lane, so
   610  				// FromBits/ToBits (x86 mask register conversions) are not needed.
   611  				if CurrentArch().Arch != "arm64" {
   612  					if err := maskFromVal.ExecuteTemplate(buffer, "maskFromVal_amd64", typeDef); err != nil {
   613  						panic(fmt.Errorf("failed to execute maskFromVal template for type %s: %w", typeDef.Name, err))
   614  					}
   615  				}
   616  			}
   617  		}
   618  	}
   619  
   620  	return buffer
   621  }
   622  
   623  type goarchFeatures struct {
   624  	// featureVar is the name of the exported feature-check variable for this
   625  	// architecture.
   626  	featureVar string
   627  
   628  	// features records per-feature information.
   629  	features map[string]featureInfo
   630  }
   631  
   632  type featureInfo struct {
   633  	// Implies is a list of other CPU features that are required for this
   634  	// feature. These are allowed to chain.
   635  	//
   636  	// For example, if the Frob feature lists "Baz", then if X.Frob() returns
   637  	// true, it must also be true that the CPU has feature Baz.
   638  	Implies []string
   639  
   640  	// Virtual means this feature is not represented directly in internal/cpu,
   641  	// but is instead the logical AND of the features in Implies.
   642  	Virtual bool
   643  }
   644  
   645  // goarchFeatureInfo maps from GOARCH to CPU feature to additional information
   646  // about that feature. Not all features need to be in this map.
   647  var goarchFeatureInfo = make(map[string]goarchFeatures)
   648  
   649  func registerFeatureInfo(goArch string, features goarchFeatures) {
   650  	goarchFeatureInfo[goArch] = features
   651  }
   652  
   653  func featureImplies(goarch string, base string) string {
   654  	// Compute the transitive closure of base.
   655  	var list []string
   656  	var visit func(f string)
   657  	visit = func(f string) {
   658  		list = append(list, f)
   659  		for _, dep := range goarchFeatureInfo[goarch].features[f].Implies {
   660  			visit(dep)
   661  		}
   662  	}
   663  	visit(base)
   664  	// Drop base
   665  	list = list[1:]
   666  	// Put in "nice" order
   667  	slices.Reverse(list)
   668  	// Combine into a comment-ready form
   669  	switch len(list) {
   670  	case 0:
   671  		return ""
   672  	case 1:
   673  		return list[0]
   674  	case 2:
   675  		return list[0] + " and " + list[1]
   676  	default:
   677  		list[len(list)-1] = "and " + list[len(list)-1]
   678  		return strings.Join(list, ", ")
   679  	}
   680  }
   681  
   682  func writeSIMDFeatures(ops []Operation) *bytes.Buffer {
   683  	// Gather all features
   684  	type featureKey struct {
   685  		GoArch  string
   686  		Feature string
   687  	}
   688  	featureSet := make(map[featureKey]struct{})
   689  	for _, op := range ops {
   690  		// Generate a feature check for each independant feature in a
   691  		// composite feature.
   692  		for feature := range strings.SplitSeq(op.CPUFeature, ",") {
   693  			feature = strings.TrimSpace(feature)
   694  			featureSet[featureKey{op.GoArch, feature}] = struct{}{}
   695  		}
   696  	}
   697  	featureKeys := slices.SortedFunc(maps.Keys(featureSet), func(a, b featureKey) int {
   698  		if c := cmp.Compare(a.GoArch, b.GoArch); c != 0 {
   699  			return c
   700  		}
   701  		return compareNatural(a.Feature, b.Feature)
   702  	})
   703  
   704  	// TODO: internal/cpu doesn't enforce these at all. You can even do
   705  	// GODEBUG=cpu.avx=off and it will happily turn off AVX without turning off
   706  	// AVX2. We need to push these dependencies into it somehow.
   707  	type feature struct {
   708  		featureKey
   709  		FeatureVar string
   710  		Virtual    bool
   711  		Implies    []string
   712  		ImpliesAll string
   713  	}
   714  	var features []feature
   715  	for _, k := range featureKeys {
   716  		featureVar := goarchFeatureInfo[k.GoArch].featureVar
   717  		fi := goarchFeatureInfo[k.GoArch].features[k.Feature]
   718  		features = append(features, feature{
   719  			featureKey: k,
   720  			FeatureVar: featureVar,
   721  			Virtual:    fi.Virtual,
   722  			Implies:    fi.Implies,
   723  			ImpliesAll: featureImplies(k.GoArch, k.Feature),
   724  		})
   725  	}
   726  
   727  	// If we ever have the same feature name on more than one GOARCH, we'll have
   728  	// to be more careful about this.
   729  	t := templateOf(simdFeaturesTemplate, "features")
   730  
   731  	buffer := new(bytes.Buffer)
   732  	buffer.WriteString(simdPackageHeader())
   733  
   734  	if err := t.Execute(buffer, features); err != nil {
   735  		panic(fmt.Errorf("failed to execute features template: %w", err))
   736  	}
   737  
   738  	return buffer
   739  }
   740  
   741  // writeSIMDStubs returns two bytes.Buffers containing the declarations for the public
   742  // and internal-use vector intrinsics.
   743  func writeSIMDStubs(ops []Operation, typeMap simdTypeMap, doDeprecatedPuns bool) (f, fI *bytes.Buffer) {
   744  	f = new(bytes.Buffer)
   745  	fI = new(bytes.Buffer)
   746  	f.WriteString(simdPackageHeader())
   747  	fI.WriteString(simdPackageHeader())
   748  
   749  	slices.SortFunc(ops, compareOperations)
   750  
   751  	for i, op := range ops {
   752  		if op.NoTypes != nil && *op.NoTypes == "true" {
   753  			continue
   754  		}
   755  		if op.SkipMaskedMethod() {
   756  			continue
   757  		}
   758  		idxVecAsScalar, err := checkVecAsScalar(op)
   759  		if err != nil {
   760  			panic(err)
   761  		}
   762  		if s, op, err := classifyOp(op); err == nil {
   763  			if op.NoGenericOps != nil && *op.NoGenericOps == "true" {
   764  				continue
   765  			}
   766  			if idxVecAsScalar != -1 {
   767  				if s == "op2" || s == "op3" || s == "op2Imm" {
   768  					s += "VecAsScalar"
   769  				} else {
   770  					panic(fmt.Errorf("simdgen only supports op2, op2Imm or op3, not %s with TreatLikeAScalarOfSize", s))
   771  				}
   772  			}
   773  			if i == 0 || op.Go != ops[i-1].Go {
   774  				if unicode.IsUpper([]rune(op.Go)[0]) {
   775  					fmt.Fprintf(f, "\n/* %s */\n", op.Go)
   776  				} else {
   777  					fmt.Fprintf(fI, "\n/* %s */\n", op.Go)
   778  				}
   779  			}
   780  			tpl := stubTemplates.Get(s)
   781  			if tpl == nil {
   782  				panic(fmt.Errorf("template %s not found", s))
   783  			}
   784  			if unicode.IsUpper([]rune(op.Go)[0]) {
   785  				if err := tpl.Execute(f, op); err != nil {
   786  					panic(fmt.Errorf("failed to execute template %s for op %v: %w", s, op, err))
   787  				}
   788  			} else {
   789  				if err := tpl.Execute(fI, op); err != nil {
   790  					panic(fmt.Errorf("failed to execute template %s for op %v: %w", s, op, err))
   791  				}
   792  			}
   793  		} else {
   794  			panic(fmt.Errorf("failed to classify op %v: %w", op.Go, err))
   795  		}
   796  	}
   797  
   798  	vectorConversions := vConvertFromTypeMap(typeMap)
   799  	for _, conv := range vectorConversions {
   800  		from, to := &conv.Tsrc, &conv.Tdst
   801  
   802  		if doDeprecatedPuns {
   803  			if err := sgutil.AsOp.Execute(f, sgutil.Conversion(from, to)); err != nil {
   804  				panic(fmt.Errorf("failed to execute vectorConversion template: %w", err))
   805  			}
   806  		}
   807  
   808  		// New style factored conversion intrinsics
   809  		if from.Name[0] != 'U' && to.Name[0] != 'U' {
   810  			continue
   811  		}
   812  		// Only emit the intrinsic if lanes are equal OR both are unsigned
   813  		if from.Lanes != to.Lanes && (from.Name[0] != 'U' || to.Name[0] != 'U') {
   814  			continue
   815  		}
   816  		switch to.Name[0] {
   817  		case 'F': // U -> F
   818  			sgutil.ToFloatsDcl.Execute(f, sgutil.Conversion(from, to))
   819  			sgutil.ToBitsDcl.Execute(f, sgutil.Conversion(to, from))
   820  		case 'I': // U -> I
   821  			sgutil.ToIntsDcl.Execute(f, sgutil.Conversion(from, to))
   822  			sgutil.ToBitsDcl.Execute(f, sgutil.Conversion(to, from))
   823  		case 'U': // U -> U
   824  			if from.Name[0] != 'U' {
   825  				continue
   826  			}
   827  			sgutil.ReshapeDcl.Execute(f, sgutil.Conversion(from, to))
   828  		default:
   829  			panic("unexpected type in reinterpret-declaration")
   830  		}
   831  	}
   832  
   833  	masks := masksFromTypeMap(typeMap)
   834  	for _, mask := range masks {
   835  		tpl := stubTemplates.Get("mask")
   836  		if tpl == nil {
   837  			panic(fmt.Errorf("template mask not found"))
   838  		}
   839  		if err := tpl.Execute(f, mask); err != nil {
   840  			panic(fmt.Errorf("failed to execute mask template for mask %s: %w", mask.Name, err))
   841  		}
   842  	}
   843  
   844  	return
   845  }
   846  

View as plain text