godefs.go

     1  // Copyright 2025 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package main
     6  
     7  import (
     8  	"fmt"
     9  	"log"
    10  	"math/rand/v2"
    11  	"regexp"
    12  	"slices"
    13  	"strconv"
    14  	"strings"
    15  	"unicode"
    16  
    17  	"simd/archsimd/_gen/unify"
    18  )
    19  
    20  type Operation struct {
    21  	rawOperation
    22  
    23  	// Go is the Go method name of this operation.
    24  	//
    25  	// It is derived from the raw Go method name by adding optional suffixes.
    26  	// Currently, "Masked" is the only suffix.
    27  	Go string
    28  
    29  	// Documentation is the doc string for this API.
    30  	//
    31  	// It is computed from the raw documentation:
    32  	//
    33  	// - "NAME" is replaced by the Go method name.
    34  	//
    35  	// - For masked operation, a sentence about masking is added.
    36  	Documentation string
    37  
    38  	// In is the sequence of parameters to the Go method.
    39  	//
    40  	// For masked operations, this will have the mask operand appended.
    41  	In []Operand
    42  }
    43  
    44  // rawOperation is the unifier representation of an [Operation]. It is
    45  // translated into a more parsed form after unifier decoding.
    46  type rawOperation struct {
    47  	Go string // Base Go method name
    48  
    49  	GoArch       string  // GOARCH for this definition
    50  	Asm          string  // Assembly mnemonic
    51  	Arrangement  *string // optional Arrangement for ARM64 SIMD operations (e.g., "4S", "2D")
    52  	OperandOrder *string // optional Operand order for better Go declarations
    53  	// Optional tag to indicate this operation is paired with special generic->machine ssa lowering rules.
    54  	// Should be paired with special templates in gen_simdrules.go
    55  	SpecialLower *string
    56  	// HiHalfAsm is the assembly mnemonic for the hi-half "2" variant of this operation,
    57  	// specified in go_arm64.yaml (e.g., "VSHRN2", "VUMULL2").
    58  	// When non-nil, simdgen generates the "2" variant machine op and folding rules.
    59  	HiHalfAsm *string
    60  
    61  	In              []Operand // Parameters
    62  	InVariant       []Operand // Optional parameters
    63  	Out             []Operand // Results
    64  	MemFeatures     *string   // The memory operand feature this operation supports
    65  	MemFeaturesData *string   // Additional data associated with MemFeatures
    66  	Commutative     bool      // Commutativity
    67  	CPUFeature      string    // CPUID/Has* feature name
    68  	Zeroing         *bool     // nil => use asm suffix ".Z"; false => do not use asm suffix ".Z"
    69  	Documentation   *string   // Documentation will be appended to the stubs comments.
    70  	AddDoc          *string   // Additional doc to be appended.
    71  	// ConstMask is a hack to reduce the size of defs the user writes for const-immediate
    72  	// If present, it will be copied to [In[0].Const].
    73  	ConstImm *string
    74  	// NameAndSizeCheck is used to check [BWDQ] maps to (8|16|32|64) elemBits.
    75  	NameAndSizeCheck *bool
    76  	// If non-nil, all generation in gen_simdTypes.go and gen_intrinsics will be skipped.
    77  	NoTypes *string
    78  	// If non-nil, all generation in gen_simdGenericOps and gen_simdrules will be skipped.
    79  	NoGenericOps *string
    80  	// If non-nil, this string will be attached to the machine ssa op name.  E.g. "const"
    81  	SSAVariant *string
    82  	// If true, do not emit method declarations, generic ops, or intrinsics for masked variants
    83  	// DO emit the architecture-specific opcodes and optimizations.
    84  	HideMaskMethods *bool
    85  }
    86  
    87  func (o *Operation) IsMasked() bool {
    88  	if len(o.InVariant) == 0 {
    89  		return false
    90  	}
    91  	if len(o.InVariant) == 1 && o.InVariant[0].Class == "mask" {
    92  		return true
    93  	}
    94  	panic(fmt.Errorf("unknown inVariant"))
    95  }
    96  
    97  func (o *Operation) SkipMaskedMethod() bool {
    98  	if o.HideMaskMethods == nil {
    99  		return false
   100  	}
   101  	if *o.HideMaskMethods && o.IsMasked() {
   102  		return true
   103  	}
   104  	return false
   105  }
   106  
   107  // hiHalfKind returns "narrow" or "long" based on whether the operation narrows or widens its elements.
   108  // Returns "" if HiHalfAsm is nil or classification is ambiguous.
   109  func (o *Operation) hiHalfKind() string {
   110  	if o.HiHalfAsm == nil {
   111  		return ""
   112  	}
   113  	// Find the first vreg input and the first vreg output to compare elemBits.
   114  	var inElemBits, outElemBits *int
   115  	for i := range o.In {
   116  		if o.In[i].Class == "vreg" && o.In[i].ElemBits != nil {
   117  			inElemBits = o.In[i].ElemBits
   118  			break
   119  		}
   120  	}
   121  	for i := range o.Out {
   122  		if o.Out[i].Class == "vreg" && o.Out[i].ElemBits != nil {
   123  			outElemBits = o.Out[i].ElemBits
   124  			break
   125  		}
   126  	}
   127  	if inElemBits == nil || outElemBits == nil {
   128  		return ""
   129  	}
   130  	if *outElemBits < *inElemBits {
   131  		return "narrow"
   132  	}
   133  	if *outElemBits > *inElemBits {
   134  		return "long"
   135  	}
   136  	return ""
   137  }
   138  
   139  var reForName = regexp.MustCompile(`\bNAME\b`)
   140  
   141  func (o *Operation) DecodeUnified(v *unify.Value) error {
   142  	if err := v.Decode(&o.rawOperation); err != nil {
   143  		return err
   144  	}
   145  
   146  	isMasked := o.IsMasked()
   147  
   148  	// Compute full Go method name.
   149  	o.Go = o.rawOperation.Go
   150  	if isMasked {
   151  		o.Go += "Masked"
   152  	}
   153  
   154  	// Compute doc string.
   155  	if o.rawOperation.Documentation != nil {
   156  		o.Documentation = *o.rawOperation.Documentation
   157  	} else {
   158  		o.Documentation = "// UNDOCUMENTED"
   159  	}
   160  	o.Documentation = reForName.ReplaceAllString(o.Documentation, o.Go)
   161  	if isMasked {
   162  		o.Documentation += "\n//\n// This operation is applied selectively under a write mask."
   163  		// Suppress generic op and method declaration for exported methods, if a mask is present.
   164  		if unicode.IsUpper([]rune(o.Go)[0]) {
   165  			trueVal := "true"
   166  			o.NoGenericOps = &trueVal
   167  			o.NoTypes = &trueVal
   168  		}
   169  	}
   170  	if o.rawOperation.AddDoc != nil {
   171  		o.Documentation += "\n" + reForName.ReplaceAllString(*o.rawOperation.AddDoc, o.Go)
   172  	}
   173  
   174  	o.In = append(o.rawOperation.In, o.rawOperation.InVariant...)
   175  
   176  	// For operations that read only the lower half of input registers (indicated by hiHalfAsm),
   177  	// add a doc note showing the compositional pattern for the upper half.
   178  	if o.rawOperation.HiHalfAsm != nil && o.hiHalfKind() == "long" {
   179  		// Count vector-register inputs (exclude immediates/scalars).
   180  		vregIns := 0
   181  		for _, in := range o.In {
   182  			if in.Class == "vreg" {
   183  				vregIns++
   184  			}
   185  		}
   186  		// note this is arm64-specific
   187  		switch vregIns {
   188  		case 2:
   189  			// Binary: MulLong, AddLong, SubLong, etc.
   190  			o.Documentation += "\n// For the high-indexed elements, use HiToLo:\n//\n//\tx.HiToLo()." + o.Go + "(y.HiToLo())"
   191  		case 1:
   192  			// Unary: ShiftLeftLongConst, etc.
   193  			o.Documentation += "\n// For the high-indexed elements, use HiToLo:\n//\n//\tx.HiToLo()." + o.Go + "(...)"
   194  		}
   195  	}
   196  
   197  	// For down conversions, the high elements are zeroed if the result has more elements.
   198  	// TODO: we should encode this logic in the YAML file, instead of hardcoding it here.
   199  	if len(o.In) > 0 && len(o.Out) > 0 {
   200  		inLanes := o.In[0].Lanes
   201  		outLanes := o.Out[0].Lanes
   202  		if inLanes != nil && outLanes != nil && *inLanes < *outLanes {
   203  			if (strings.Contains(o.Go, "Saturate") || strings.Contains(o.Go, "TruncTo")) &&
   204  				!strings.Contains(o.Go, "Concat") {
   205  				o.Documentation += "\n// Results are packed to low elements in the returned vector, its upper elements are zeroed."
   206  			}
   207  		}
   208  	}
   209  
   210  	return nil
   211  }
   212  
   213  func (o *Operation) VectorWidth() int {
   214  	out := o.Out[0]
   215  	if out.Class == "vreg" {
   216  		return *out.Bits
   217  	} else if out.Class == "greg" || out.Class == "mask" {
   218  		for i := range o.In {
   219  			if o.In[i].Class == "vreg" {
   220  				return *o.In[i].Bits
   221  			}
   222  		}
   223  	}
   224  	panic(fmt.Errorf("Figure out what the vector width is for %v and implement it", *o))
   225  }
   226  
   227  // Right now simdgen computes the machine op name for most instructions
   228  // as $Name$OutputSize, by this denotation, these instructions are "overloaded".
   229  // for example:
   230  // (Uint16x8) ConvertToInt8
   231  // (Uint16x16) ConvertToInt8
   232  // are both VPMOVWB128.
   233  // To make them distinguishable we need to append the input size to them as well.
   234  // TODO: document them well in the generated code.
   235  var demotingConvertOps = map[string]bool{
   236  	"VPMOVQD128": true, "VPMOVSQD128": true, "VPMOVUSQD128": true, "VPMOVQW128": true, "VPMOVSQW128": true,
   237  	"VPMOVUSQW128": true, "VPMOVDW128": true, "VPMOVSDW128": true, "VPMOVUSDW128": true, "VPMOVQB128": true,
   238  	"VPMOVSQB128": true, "VPMOVUSQB128": true, "VPMOVDB128": true, "VPMOVSDB128": true, "VPMOVUSDB128": true,
   239  	"VPMOVWB128": true, "VPMOVSWB128": true, "VPMOVUSWB128": true,
   240  	"VPMOVQDMasked128": true, "VPMOVSQDMasked128": true, "VPMOVUSQDMasked128": true, "VPMOVQWMasked128": true, "VPMOVSQWMasked128": true,
   241  	"VPMOVUSQWMasked128": true, "VPMOVDWMasked128": true, "VPMOVSDWMasked128": true, "VPMOVUSDWMasked128": true, "VPMOVQBMasked128": true,
   242  	"VPMOVSQBMasked128": true, "VPMOVUSQBMasked128": true, "VPMOVDBMasked128": true, "VPMOVSDBMasked128": true, "VPMOVUSDBMasked128": true,
   243  	"VPMOVWBMasked128": true, "VPMOVSWBMasked128": true, "VPMOVUSWBMasked128": true,
   244  }
   245  
   246  func machineOpName(maskType maskShape, gOp Operation) string {
   247  	asm := gOp.Asm
   248  	if maskType == OneMask {
   249  		asm += "Masked"
   250  	}
   251  	// For ARM64, use arrangement to create distinct SSA op names
   252  	if gOp.Arrangement != nil && *gOp.Arrangement != "" {
   253  		asm = fmt.Sprintf("%s%s", asm, *gOp.Arrangement)
   254  	} else {
   255  		asm = fmt.Sprintf("%s%d", asm, gOp.VectorWidth())
   256  	}
   257  	if gOp.SSAVariant != nil {
   258  		asm += *gOp.SSAVariant
   259  	}
   260  	if demotingConvertOps[asm] {
   261  		// Need to append the size of the source as well.
   262  		// TODO: should be "%sto%d".
   263  		asm = fmt.Sprintf("%s_%d", asm, *gOp.In[0].Bits)
   264  	}
   265  	return asm
   266  }
   267  
   268  func compareStringPointers(x, y *string) int {
   269  	if x != nil && y != nil {
   270  		return compareNatural(*x, *y)
   271  	}
   272  	if x == nil && y == nil {
   273  		return 0
   274  	}
   275  	if x == nil {
   276  		return -1
   277  	}
   278  	return 1
   279  }
   280  
   281  func compareIntPointers(x, y *int) int {
   282  	if x != nil && y != nil {
   283  		return *x - *y
   284  	}
   285  	if x == nil && y == nil {
   286  		return 0
   287  	}
   288  	if x == nil {
   289  		return -1
   290  	}
   291  	return 1
   292  }
   293  
   294  func compareOperations(x, y Operation) int {
   295  	if c := compareNatural(x.Go, y.Go); c != 0 {
   296  		return c
   297  	}
   298  	xIn, yIn := x.In, y.In
   299  
   300  	if len(xIn) > len(yIn) && xIn[len(xIn)-1].Class == "mask" {
   301  		xIn = xIn[:len(xIn)-1]
   302  	} else if len(xIn) < len(yIn) && yIn[len(yIn)-1].Class == "mask" {
   303  		yIn = yIn[:len(yIn)-1]
   304  	}
   305  
   306  	if len(xIn) < len(yIn) {
   307  		return -1
   308  	}
   309  	if len(xIn) > len(yIn) {
   310  		return 1
   311  	}
   312  	if len(x.Out) < len(y.Out) {
   313  		return -1
   314  	}
   315  	if len(x.Out) > len(y.Out) {
   316  		return 1
   317  	}
   318  	for i := range xIn {
   319  		ox, oy := &xIn[i], &yIn[i]
   320  		if c := compareOperands(ox, oy); c != 0 {
   321  			return c
   322  		}
   323  	}
   324  	return 0
   325  }
   326  
   327  func compareOperands(x, y *Operand) int {
   328  	if c := compareNatural(x.Class, y.Class); c != 0 {
   329  		return c
   330  	}
   331  	if x.Class == "immediate" {
   332  		return compareStringPointers(x.ImmOffset, y.ImmOffset)
   333  	} else {
   334  		if c := compareStringPointers(x.Base, y.Base); c != 0 {
   335  			return c
   336  		}
   337  		if c := compareIntPointers(x.ElemBits, y.ElemBits); c != 0 {
   338  			return c
   339  		}
   340  		if c := compareIntPointers(x.Bits, y.Bits); c != 0 {
   341  			return c
   342  		}
   343  		if c := compareIntPointers(x.ListNumber, y.ListNumber); c != 0 {
   344  			return c
   345  		}
   346  		return 0
   347  	}
   348  }
   349  
   350  type Operand struct {
   351  	Class string // One of "mask", "immediate", "vreg", "greg", and "mem"
   352  
   353  	Go     *string // Go type of this operand
   354  	AsmPos int     // Position of this operand in the assembly instruction
   355  
   356  	Base     *string // Base Go type ("int", "uint", "float")
   357  	ElemBits *int    // Element bit width
   358  	Bits     *int    // Total vector bit width
   359  
   360  	Const *string // Optional constant value for immediates.
   361  	// Optional immediate arg offsets. If this field is non-nil,
   362  	// This operand will be an immediate operand:
   363  	// The compiler will right-shift the user-passed value by ImmOffset and set it as the AuxInt
   364  	// field of the operation.
   365  	ImmOffset *string
   366  	ImmMax    *int    // optional maximum immediate, also highest case in immediate jump table
   367  	Name      *string // optional name in the Go intrinsic declaration
   368  	Lanes     *int    // *Lanes equals Bits/ElemBits except for scalars, when *Lanes == 1
   369  	// TreatLikeAScalarOfSize means only the lower $TreatLikeAScalarOfSize bits of the vector
   370  	// is used, so at the API level we can make it just a scalar value of this size; Then we
   371  	// can overwrite it to a vector of the right size during intrinsics stage.
   372  	TreatLikeAScalarOfSize *int
   373  	// If non-nil, it means the [Class] field is overwritten here, right now this is used to
   374  	// overwrite the results of AVX2 compares to masks.
   375  	OverwriteClass *string
   376  	// If non-nil, it means the [Base] field is overwritten here. This field exist solely
   377  	// because Intel's XED data is inconsistent. e.g. VANDNP[SD] marks its operand int.
   378  	OverwriteBase *string
   379  	// If non-nil, it means the [ElementBits] field is overwritten. This field exist solely
   380  	// because Intel's XED data is inconsistent. e.g. AVX512 VPMADDUBSW marks its operand
   381  	// elemBits 16, which should be 8.
   382  	OverwriteElementBits *int
   383  	// For greg only, specifically VPEXTR[BW], their results are specified by Intel as 32 bits,
   384  	// but they really are 8/16 bits.
   385  	OverwriteBits *int
   386  	// FixedReg is the name of the fixed registers
   387  	FixedReg *string
   388  	// If non-nil, marks this vreg as a register list operand (for TBL/TBX).
   389  	// Currently only list number 0 is supported (we might need to teach regalloc handle register lists
   390  	// to support more than one register in the list).
   391  	ListNumber *int
   392  }
   393  
   394  // isDigit returns true if the byte is an ASCII digit.
   395  func isDigit(b byte) bool {
   396  	return b >= '0' && b <= '9'
   397  }
   398  
   399  // compareNatural performs a "natural sort" comparison of two strings.
   400  // It compares non-digit sections lexicographically and digit sections
   401  // numerically.  In the case of string-unequal "equal" strings like
   402  // "a01b" and "a1b", strings.Compare breaks the tie.
   403  //
   404  // It returns:
   405  //
   406  //	-1 if s1 < s2
   407  //	 0 if s1 == s2
   408  //	+1 if s1 > s2
   409  func compareNatural(s1, s2 string) int {
   410  	i, j := 0, 0
   411  	len1, len2 := len(s1), len(s2)
   412  
   413  	for i < len1 && j < len2 {
   414  		// Find a non-digit segment or a number segment in both strings.
   415  		if isDigit(s1[i]) && isDigit(s2[j]) {
   416  			// Number segment comparison.
   417  			numStart1 := i
   418  			for i < len1 && isDigit(s1[i]) {
   419  				i++
   420  			}
   421  			num1, _ := strconv.Atoi(s1[numStart1:i])
   422  
   423  			numStart2 := j
   424  			for j < len2 && isDigit(s2[j]) {
   425  				j++
   426  			}
   427  			num2, _ := strconv.Atoi(s2[numStart2:j])
   428  
   429  			if num1 < num2 {
   430  				return -1
   431  			}
   432  			if num1 > num2 {
   433  				return 1
   434  			}
   435  			// "1" < "01".  Don't expect it in simdgen, but just in case.
   436  			if ln1, ln2 := i-numStart1, j-numStart2; ln1 != ln2 {
   437  				return ln1 - ln2
   438  			}
   439  			// If numbers are equal, continue to the next segment.
   440  		} else {
   441  			// Non-digit comparison.
   442  			if s1[i] < s2[j] {
   443  				return -1
   444  			}
   445  			if s1[i] > s2[j] {
   446  				return 1
   447  			}
   448  			i++
   449  			j++
   450  		}
   451  	}
   452  
   453  	// deal with a01b vs a1b; there needs to be an order.
   454  	return strings.Compare(s1, s2)
   455  }
   456  
   457  // generatedHeader returns the architecture-specific header for generated files.
   458  func generatedHeader() string {
   459  	return CurrentArch().GeneratedHeader
   460  }
   461  
   462  func writeGoDefs(path string, cl unify.Closure) error {
   463  	// TODO: Merge operations with the same signature but multiple
   464  	// implementations (e.g., SSE vs AVX)
   465  	var ops []Operation
   466  	for def := range cl.All() {
   467  		var op Operation
   468  		if !def.Exact() {
   469  			continue
   470  		}
   471  		if err := def.Decode(&op); err != nil {
   472  			log.Println(err.Error())
   473  			log.Println(def)
   474  			continue
   475  		}
   476  		// TODO: verify that this is safe.
   477  		op.sortOperand()
   478  		op.adjustAsm()
   479  		ops = append(ops, op)
   480  	}
   481  
   482  	rand.Shuffle(len(ops), func(i, j int) {
   483  		ops[i], ops[j] = ops[j], ops[i]
   484  	})
   485  
   486  	slices.SortFunc(ops, compareOperations)
   487  	// The parsed XED data might contain duplicates, like
   488  	// 512 bits VPADDP.
   489  	deduped := dedup(ops)
   490  	slices.SortFunc(deduped, compareOperations)
   491  
   492  	if *Verbose {
   493  		log.Printf("dedup len: %d, ops len: %d\n", len(deduped), len(ops))
   494  	}
   495  	var err error
   496  	if err = overwrite(deduped); err != nil {
   497  		return err
   498  	}
   499  	if *Verbose {
   500  		log.Printf("dedup len: %d\n", len(deduped))
   501  	}
   502  	if !*FlagNoDedup {
   503  		// TODO: This can hide mistakes in the API definitions, especially when
   504  		// multiple patterns result in the same API unintentionally. Make it stricter.
   505  		if deduped, err = dedupGodef(deduped); err != nil {
   506  			return err
   507  		}
   508  	}
   509  	if *Verbose {
   510  		log.Printf("dedup len: %d\n", len(deduped))
   511  	}
   512  	if !*FlagNoConstImmPorting {
   513  		if err = copyConstImm(deduped); err != nil {
   514  			return err
   515  		}
   516  	}
   517  	if *Verbose {
   518  		log.Printf("dedup len: %d\n", len(deduped))
   519  	}
   520  	reportXEDInconsistency(deduped)
   521  
   522  	// Sorting again, just in case.
   523  	slices.SortFunc(deduped, compareOperations)
   524  
   525  	typeMap := parseSIMDTypes(deduped)
   526  
   527  	archInfo := CurrentArch()
   528  	archLower := archInfo.Arch
   529  	archUpper := archInfo.ArchUpper
   530  
   531  	formatWriteAndClose(writeSIMDTypes(typeMap), path, "src/"+simdPackage+"/types_"+archLower+".go")
   532  	// TODO: Enable CPU feature generation for non-x86 architectures.
   533  	if archLower == "amd64" {
   534  		formatWriteAndClose(writeSIMDFeatures(deduped), path, "src/"+simdPackage+"/cpu.go")
   535  	}
   536  	f, fI := writeSIMDStubs(deduped, typeMap, archLower == "amd64")
   537  	formatWriteAndClose(f, path, "src/"+simdPackage+"/ops_"+archLower+".go")
   538  	formatWriteAndClose(fI, path, "src/"+simdPackage+"/ops_internal_"+archLower+".go")
   539  	formatWriteAndClose(writeSIMDIntrinsics(deduped, typeMap), path, "src/cmd/compile/internal/ssagen/simd"+archUpper+"intrinsics.go")
   540  	const simdGenericOpsFile = "src/cmd/compile/internal/ssa/_gen/simdgenericOps.go"
   541  	formatWriteAndClose(writeSIMDGenericOps(deduped, path+"/"+simdGenericOpsFile), path, simdGenericOpsFile)
   542  	formatWriteAndClose(writeSIMDMachineOps(deduped), path, "src/cmd/compile/internal/ssa/_gen/simd"+archUpper+"ops.go")
   543  	formatWriteAndClose(writeSIMDSSA(deduped), path, "src/cmd/compile/internal/"+archLower+"/simdssa.go")
   544  	writeAndClose(writeSIMDRules(deduped).Bytes(), path, "src/cmd/compile/internal/ssa/_gen/simd"+archUpper+".rules")
   545  
   546  	return nil
   547  }
   548
View as plain text