Source file src/simd/archsimd/_gen/simdgen/xed.go

     1  // Copyright 2025 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package main
     6  
     7  import (
     8  	"fmt"
     9  	"log"
    10  	"maps"
    11  	"reflect"
    12  	"regexp"
    13  	"slices"
    14  	"strconv"
    15  	"strings"
    16  
    17  	"simd/archsimd/_gen/unify"
    18  
    19  	"golang.org/x/arch/x86/xeddata"
    20  	"gopkg.in/yaml.v3"
    21  )
    22  
    23  const (
    24  	NOT_REG_CLASS = iota // not a register
    25  	VREG_CLASS           // classify as a vector register; see
    26  	GREG_CLASS           // classify as a general register
    27  )
    28  
    29  // instVariant is a bitmap indicating a variant of an instruction that has
    30  // optional parameters.
    31  type instVariant uint8
    32  
    33  const (
    34  	instVariantNone instVariant = 0
    35  
    36  	// instVariantMasked indicates that this is the masked variant of an
    37  	// optionally-masked instruction.
    38  	instVariantMasked instVariant = 1 << iota
    39  )
    40  
    41  var operandRemarks int
    42  
    43  var skipMemOpsInstrs = map[string]bool{
    44  	// The SHA instructions has confusing semantics which we are too complicated
    45  	// to support.
    46  	"SHA1MSG1":    true,
    47  	"SHA1MSG2":    true,
    48  	"SHA1RNDS4":   true,
    49  	"SHA1NEXTE":   true,
    50  	"SHA256MSG1":  true,
    51  	"SHA256MSG2":  true,
    52  	"SHA256RNDS2": true,
    53  	// We don't support 3 input instructions with a memory operand (this appears
    54  	// to be the only one).
    55  	"VPBLENDVB": true,
    56  }
    57  
    58  // TODO: Doc. Returns Values with Def domains.
    59  func loadXED(xedPath string) []*unify.Value {
    60  	// TODO: Obviously a bunch more to do here.
    61  
    62  	db, err := xeddata.NewDatabase(xedPath)
    63  	if err != nil {
    64  		log.Fatalf("open database: %v", err)
    65  	}
    66  
    67  	var defs []*unify.Value
    68  	type opData struct {
    69  		inst *xeddata.Inst
    70  		ops  []operand
    71  		mem  string
    72  	}
    73  	// Maps from opcode to opdata(s).
    74  	memOps := make(map[string][]opData, 0)
    75  	otherOps := make(map[string][]opData, 0)
    76  	appendDefs := func(inst *xeddata.Inst, ops []operand, addFields map[string]string) {
    77  		applyQuirks(inst, ops)
    78  
    79  		defsPos := len(defs)
    80  		defs = append(defs, instToUVal(inst, ops, addFields)...)
    81  
    82  		if *flagDebugXED {
    83  			for i := defsPos; i < len(defs); i++ {
    84  				y, _ := yaml.Marshal(defs[i])
    85  				fmt.Printf("==>\n%s\n", y)
    86  			}
    87  		}
    88  	}
    89  	err = xeddata.WalkInsts(xedPath, func(inst *xeddata.Inst) {
    90  		inst.Pattern = xeddata.ExpandStates(db, inst.Pattern)
    91  
    92  		switch {
    93  		case inst.RealOpcode == "N":
    94  			return // Skip unstable instructions
    95  		case !(strings.HasPrefix(inst.Extension, "AVX") || strings.HasPrefix(inst.Extension, "SHA") ||
    96  			inst.Extension == "FMA" || inst.Extension == "VAES"):
    97  			// We're only interested in AVX and SHA instructions.
    98  			return
    99  		}
   100  
   101  		if *flagDebugXED {
   102  			fmt.Printf("%s:\n%+v\n", inst.Pos, inst)
   103  		}
   104  
   105  		ops, err := decodeOperands(db, strings.Fields(inst.Operands))
   106  		if err != nil {
   107  			operandRemarks++
   108  			if *Verbose {
   109  				log.Printf("%s: [%s] %s", inst.Pos, inst.Opcode(), err)
   110  			}
   111  			return
   112  		}
   113  		var data map[string][]opData
   114  		opcode := inst.Opcode()
   115  		mem := checkMem(ops)
   116  		if mem == "hasMem" && !skipMemOpsInstrs[opcode] {
   117  			// A pure vreg variant might exist, wait for later to see if we can
   118  			// merge them
   119  			data = memOps
   120  		} else {
   121  			data = otherOps
   122  		}
   123  		if _, ok := data[opcode]; !ok {
   124  			s := make([]opData, 1)
   125  			s[0] = opData{inst, ops, mem}
   126  			data[opcode] = s
   127  		} else {
   128  			data[opcode] = append(data[opcode], opData{inst, ops, mem})
   129  		}
   130  	})
   131  	for _, s := range otherOps {
   132  		for _, o := range s {
   133  			addFields := map[string]string{}
   134  			if o.mem == "noMem" {
   135  				opcode := o.inst.Opcode()
   136  				// Checking if there is a vbcst variant of this operation exist
   137  				// First check the opcode
   138  				// Keep this logic in sync with [decodeOperands]
   139  				if ms, ok := memOps[opcode]; ok && !strings.HasPrefix(opcode, "VMOV") {
   140  					feat1, ok1 := decodeCPUFeature(o.inst)
   141  					// Then check if there exist such an operation that for all vreg
   142  					// shapes they are the same at the same index
   143  					var feat1Match, feat2Match string
   144  					matchIdx := -1
   145  					var featMismatchCnt int
   146  				outer:
   147  					for i, m := range ms {
   148  						// Their CPU feature should match first
   149  						var featMismatch bool
   150  						feat2, ok2 := decodeCPUFeature(m.inst)
   151  						if !ok1 || !ok2 {
   152  							continue
   153  						}
   154  						if feat1 != feat2 {
   155  							featMismatch = true
   156  							featMismatchCnt++
   157  						}
   158  						if len(o.ops) == len(m.ops) {
   159  							for j := range o.ops {
   160  								if reflect.TypeOf(o.ops[j]) == reflect.TypeOf(m.ops[j]) {
   161  									v1, ok3 := o.ops[j].(operandVReg)
   162  									v2, _ := m.ops[j].(operandVReg)
   163  									if !ok3 {
   164  										continue
   165  									}
   166  									if v1.vecShape != v2.vecShape {
   167  										// A mismatch, skip this memOp
   168  										continue outer
   169  									}
   170  								} else {
   171  									_, ok3 := o.ops[j].(operandVReg)
   172  									_, ok4 := m.ops[j].(operandMem)
   173  									// The only difference must be the vreg and mem, and the operand must be a read operand.
   174  									if !ok3 || !ok4 || !o.ops[j].common().action.r {
   175  										// A mismatch, skip this memOp
   176  										continue outer
   177  									}
   178  								}
   179  							}
   180  							// Found a match, break early
   181  							matchIdx = i
   182  							feat1Match = feat1
   183  							feat2Match = feat2
   184  							if featMismatchCnt > 1 {
   185  								panic(fmt.Sprintf("multiple feature mismatch vbcst memops detected for %s, simdgen failed to distinguish", opcode))
   186  							}
   187  							if !featMismatch {
   188  								// Mismatch feat is ok but should prioritize matching cases.
   189  								break
   190  							}
   191  						}
   192  					}
   193  					// Remove the match from memOps, it's now merged to this pure vreg operation
   194  					if matchIdx != -1 {
   195  						memOps[opcode] = append(memOps[opcode][:matchIdx], memOps[opcode][matchIdx+1:]...)
   196  						// Merge is done by adding a new field
   197  						// Right now we only have vbcst
   198  						addFields["memFeatures"] = "vbcst"
   199  						if feat1Match != feat2Match {
   200  							addFields["memFeaturesData"] = fmt.Sprintf("feat1=%s;feat2=%s", feat1Match, feat2Match)
   201  						}
   202  					}
   203  				}
   204  			}
   205  			appendDefs(o.inst, o.ops, addFields)
   206  		}
   207  	}
   208  	for _, ms := range memOps {
   209  		for _, m := range ms {
   210  			if *Verbose {
   211  				log.Printf("mem op not merged: %s, %v\n", m.inst.Opcode(), m)
   212  			}
   213  			appendDefs(m.inst, m.ops, nil)
   214  		}
   215  	}
   216  	if err != nil {
   217  		log.Fatalf("walk insts: %v", err)
   218  	}
   219  
   220  	if len(unknownFeatures) > 0 {
   221  		if !*Verbose {
   222  			nInst := 0
   223  			for _, insts := range unknownFeatures {
   224  				nInst += len(insts)
   225  			}
   226  			log.Printf("%d unhandled CPU features for %d instructions (use -v for details)", len(unknownFeatures), nInst)
   227  		} else {
   228  			keys := slices.Sorted(maps.Keys(unknownFeatures))
   229  			for _, key := range keys {
   230  				log.Printf("unhandled ISASet %s", key)
   231  				log.Printf("  opcodes: %s", slices.Sorted(maps.Keys(unknownFeatures[key])))
   232  			}
   233  		}
   234  	}
   235  
   236  	return defs
   237  }
   238  
   239  var (
   240  	maskRequiredRe = regexp.MustCompile(`VPCOMPRESS[BWDQ]|VCOMPRESSP[SD]|VPEXPAND[BWDQ]|VEXPANDP[SD]`)
   241  	maskOptionalRe = regexp.MustCompile(`VPCMP(EQ|GT|U)?[BWDQ]|VCMPP[SD]`)
   242  )
   243  
   244  func applyQuirks(inst *xeddata.Inst, ops []operand) {
   245  	opc := inst.Opcode()
   246  	switch {
   247  	case maskRequiredRe.MatchString(opc):
   248  		// The mask on these instructions is marked optional, but the
   249  		// instruction is pointless without the mask.
   250  		for i, op := range ops {
   251  			if op, ok := op.(operandMask); ok {
   252  				op.optional = false
   253  				ops[i] = op
   254  			}
   255  		}
   256  
   257  	case maskOptionalRe.MatchString(opc):
   258  		// Conversely, these masks should be marked optional and aren't.
   259  		for i, op := range ops {
   260  			if op, ok := op.(operandMask); ok && op.action.r {
   261  				op.optional = true
   262  				ops[i] = op
   263  			}
   264  		}
   265  	}
   266  }
   267  
   268  type operandCommon struct {
   269  	action operandAction
   270  }
   271  
   272  // operandAction defines whether this operand is read and/or written.
   273  //
   274  // TODO: Should this live in [xeddata.Operand]?
   275  type operandAction struct {
   276  	r  bool // Read
   277  	w  bool // Written
   278  	cr bool // Read is conditional (implies r==true)
   279  	cw bool // Write is conditional (implies w==true)
   280  }
   281  
   282  type operandMem struct {
   283  	operandCommon
   284  	vecShape
   285  	elemBaseType scalarBaseType
   286  	// The following fields are not flushed to the final output
   287  	// Supports full-vector broadcasting; implies the operand having a "vv"(vector vector) type specified in width and
   288  	// the instruction is with attribute TXT=BCASTSTR.
   289  	vbcst   bool
   290  	unknown bool // unknown kind
   291  }
   292  
   293  type vecShape struct {
   294  	elemBits  int    // Element size in bits
   295  	bits      int    // Register width in bits (total vector bits)
   296  	fixedName string // the fixed register name
   297  }
   298  
   299  type operandVReg struct { // Vector register
   300  	operandCommon
   301  	vecShape
   302  	elemBaseType scalarBaseType
   303  }
   304  
   305  type operandGReg struct { // Vector register
   306  	operandCommon
   307  	vecShape
   308  	elemBaseType scalarBaseType
   309  }
   310  
   311  // operandMask is a vector mask.
   312  //
   313  // Regardless of the actual mask representation, the [vecShape] of this operand
   314  // corresponds to the "bit for bit" type of mask. That is, elemBits gives the
   315  // element width covered by each mask element, and bits/elemBits gives the total
   316  // number of mask elements. (bits gives the total number of bits as if this were
   317  // a bit-for-bit mask, which may be meaningless on its own.)
   318  type operandMask struct {
   319  	operandCommon
   320  	vecShape
   321  	// Bits in the mask is w/bits.
   322  
   323  	allMasks bool // If set, size cannot be inferred because all operands are masks.
   324  
   325  	// Mask can be omitted, in which case it defaults to K0/"no mask"
   326  	optional bool
   327  }
   328  
   329  type operandImm struct {
   330  	operandCommon
   331  	bits int // Immediate size in bits
   332  }
   333  
   334  type operand interface {
   335  	common() operandCommon
   336  	addToDef(b *unify.DefBuilder)
   337  }
   338  
   339  func strVal(s any) *unify.Value {
   340  	return unify.NewValue(unify.NewStringExact(fmt.Sprint(s)))
   341  }
   342  
   343  func (o operandCommon) common() operandCommon {
   344  	return o
   345  }
   346  
   347  func (o operandMem) addToDef(b *unify.DefBuilder) {
   348  	b.Add("class", strVal("memory"))
   349  	if o.unknown {
   350  		return
   351  	}
   352  	baseDomain, err := unify.NewStringRegex(o.elemBaseType.regex())
   353  	if err != nil {
   354  		panic("parsing baseRe: " + err.Error())
   355  	}
   356  	b.Add("base", unify.NewValue(baseDomain))
   357  	b.Add("bits", strVal(o.bits))
   358  	if o.elemBits != o.bits {
   359  		b.Add("elemBits", strVal(o.elemBits))
   360  	}
   361  }
   362  
   363  func (o operandVReg) addToDef(b *unify.DefBuilder) {
   364  	baseDomain, err := unify.NewStringRegex(o.elemBaseType.regex())
   365  	if err != nil {
   366  		panic("parsing baseRe: " + err.Error())
   367  	}
   368  	b.Add("class", strVal("vreg"))
   369  	b.Add("bits", strVal(o.bits))
   370  	b.Add("base", unify.NewValue(baseDomain))
   371  	// If elemBits == bits, then the vector can be ANY shape. This happens with,
   372  	// for example, logical ops.
   373  	if o.elemBits != o.bits {
   374  		b.Add("elemBits", strVal(o.elemBits))
   375  	}
   376  	if o.fixedName != "" {
   377  		b.Add("fixedReg", strVal(o.fixedName))
   378  	}
   379  }
   380  
   381  func (o operandGReg) addToDef(b *unify.DefBuilder) {
   382  	baseDomain, err := unify.NewStringRegex(o.elemBaseType.regex())
   383  	if err != nil {
   384  		panic("parsing baseRe: " + err.Error())
   385  	}
   386  	b.Add("class", strVal("greg"))
   387  	b.Add("bits", strVal(o.bits))
   388  	b.Add("base", unify.NewValue(baseDomain))
   389  	if o.elemBits != o.bits {
   390  		b.Add("elemBits", strVal(o.elemBits))
   391  	}
   392  	if o.fixedName != "" {
   393  		b.Add("fixedReg", strVal(o.fixedName))
   394  	}
   395  }
   396  
   397  func (o operandMask) addToDef(b *unify.DefBuilder) {
   398  	b.Add("class", strVal("mask"))
   399  	if o.allMasks {
   400  		// If all operands are masks, omit sizes and let unification determine mask sizes.
   401  		return
   402  	}
   403  	b.Add("elemBits", strVal(o.elemBits))
   404  	b.Add("bits", strVal(o.bits))
   405  	if o.fixedName != "" {
   406  		b.Add("fixedReg", strVal(o.fixedName))
   407  	}
   408  }
   409  
   410  func (o operandImm) addToDef(b *unify.DefBuilder) {
   411  	b.Add("class", strVal("immediate"))
   412  	b.Add("bits", strVal(o.bits))
   413  }
   414  
   415  var actionEncoding = map[string]operandAction{
   416  	"r":   {r: true},
   417  	"cr":  {r: true, cr: true},
   418  	"w":   {w: true},
   419  	"cw":  {w: true, cw: true},
   420  	"rw":  {r: true, w: true},
   421  	"crw": {r: true, w: true, cr: true},
   422  	"rcw": {r: true, w: true, cw: true},
   423  }
   424  
   425  func decodeOperand(db *xeddata.Database, operand string) (operand, error) {
   426  	op, err := xeddata.NewOperand(db, operand)
   427  	if err != nil {
   428  		log.Fatalf("parsing operand %q: %v", operand, err)
   429  	}
   430  	if *flagDebugXED {
   431  		fmt.Printf("  %+v\n", op)
   432  	}
   433  
   434  	if strings.HasPrefix(op.Name, "EMX_BROADCAST") {
   435  		// This refers to a set of macros defined in all-state.txt that set a
   436  		// BCAST operand to various fixed values. But the BCAST operand is
   437  		// itself suppressed and "internal", so I think we can just ignore this
   438  		// operand.
   439  		return nil, nil
   440  	}
   441  
   442  	// TODO: See xed_decoded_inst_operand_action. This might need to be more
   443  	// complicated.
   444  	action, ok := actionEncoding[op.Action]
   445  	if !ok {
   446  		return nil, fmt.Errorf("unknown action %q", op.Action)
   447  	}
   448  	common := operandCommon{action: action}
   449  
   450  	lhs := op.NameLHS()
   451  	if strings.HasPrefix(lhs, "MEM") {
   452  		// looks like XED data has an inconsistency on VPADDD, marking attribute
   453  		// VPBROADCASTD instead of the canonical BCASTSTR.
   454  		if op.Width == "vv" && (op.Attributes["TXT=BCASTSTR"] ||
   455  			op.Attributes["TXT=VPBROADCASTD"]) {
   456  			baseType, elemBits, ok := decodeType(op)
   457  			if !ok {
   458  				return nil, fmt.Errorf("failed to decode memory width %q", operand)
   459  			}
   460  			// This operand has two possible width([bits]):
   461  			// 1. the same as the other operands
   462  			// 2. the element width as the other operands (broaccasting)
   463  			// left it default to 2, later we will set a new field in the operation
   464  			// to indicate this dual-width property.
   465  			shape := vecShape{elemBits: elemBits, bits: elemBits}
   466  			return operandMem{
   467  				operandCommon: common,
   468  				vecShape:      shape,
   469  				elemBaseType:  baseType,
   470  				vbcst:         true,
   471  				unknown:       false,
   472  			}, nil
   473  		} else {
   474  			baseType, elemBits, ok := decodeType(op)
   475  			if !ok {
   476  				return nil, fmt.Errorf("failed to decode memory width %q", operand)
   477  			}
   478  			sizeStr := db.WidthSize(op.Width, xeddata.OpSize64)
   479  			bytes, err := strconv.Atoi(sizeStr)
   480  			if err != nil {
   481  				return nil, fmt.Errorf("failed to decode memory width %q: %s", operand, err)
   482  			}
   483  			if bytes > 0 {
   484  				memBits := bytes * 8
   485  				shape := vecShape{elemBits: elemBits, bits: memBits}
   486  				return operandMem{
   487  					operandCommon: common,
   488  					vecShape:      shape,
   489  					elemBaseType:  baseType,
   490  					vbcst:         false,
   491  					unknown:       false,
   492  				}, nil
   493  			}
   494  		}
   495  		return operandMem{
   496  			operandCommon: common,
   497  			unknown:       true,
   498  		}, nil
   499  	} else if strings.HasPrefix(lhs, "REG") {
   500  		if op.Width == "mskw" {
   501  			// The mask operand doesn't specify a width. We have to infer it.
   502  			//
   503  			// XED uses the marker ZEROSTR to indicate that a mask operand is
   504  			// optional and, if omitted, implies K0, aka "no mask".
   505  			return operandMask{
   506  				operandCommon: common,
   507  				optional:      op.Attributes["TXT=ZEROSTR"],
   508  			}, nil
   509  		} else {
   510  			class, regBits, fixedReg := decodeReg(op)
   511  			if class == NOT_REG_CLASS {
   512  				return nil, fmt.Errorf("failed to decode register %q", operand)
   513  			}
   514  			baseType, elemBits, ok := decodeType(op)
   515  			if !ok {
   516  				return nil, fmt.Errorf("failed to decode register width %q", operand)
   517  			}
   518  			shape := vecShape{elemBits: elemBits, bits: regBits, fixedName: fixedReg}
   519  			if class == VREG_CLASS {
   520  				return operandVReg{
   521  					operandCommon: common,
   522  					vecShape:      shape,
   523  					elemBaseType:  baseType,
   524  				}, nil
   525  			}
   526  			// general register
   527  			m := min(shape.bits, shape.elemBits)
   528  			shape.bits, shape.elemBits = m, m
   529  			return operandGReg{
   530  				operandCommon: common,
   531  				vecShape:      shape,
   532  				elemBaseType:  baseType,
   533  			}, nil
   534  
   535  		}
   536  	} else if strings.HasPrefix(lhs, "IMM") {
   537  		_, bits, ok := decodeType(op)
   538  		if !ok {
   539  			return nil, fmt.Errorf("failed to decode register width %q", operand)
   540  		}
   541  		return operandImm{
   542  			operandCommon: common,
   543  			bits:          bits,
   544  		}, nil
   545  	}
   546  
   547  	// TODO: BASE and SEG
   548  	return nil, fmt.Errorf("unknown operand LHS %q in %q", lhs, operand)
   549  }
   550  
   551  func decodeOperands(db *xeddata.Database, operands []string) (ops []operand, err error) {
   552  	// Decode the XED operand descriptions.
   553  	for _, o := range operands {
   554  		op, err := decodeOperand(db, o)
   555  		if err != nil {
   556  			return nil, err
   557  		}
   558  		if op != nil {
   559  			ops = append(ops, op)
   560  		}
   561  	}
   562  
   563  	// XED doesn't encode the size of mask operands. If there are mask operands,
   564  	// try to infer their sizes from other operands.
   565  	if err := inferMaskSizes(ops); err != nil {
   566  		return nil, fmt.Errorf("%w in operands %+v", err, operands)
   567  	}
   568  
   569  	return ops, nil
   570  }
   571  
   572  func inferMaskSizes(ops []operand) error {
   573  	// This is a heuristic and it falls apart in some cases:
   574  	//
   575  	// - Mask operations like KAND[BWDQ] have *nothing* in the XED to indicate
   576  	// mask size.
   577  	//
   578  	// - VINSERT*, VPSLL*, VPSRA*, and VPSRL* and some others naturally have
   579  	// mixed input sizes and the XED doesn't indicate which operands the mask
   580  	// applies to.
   581  	//
   582  	// - VPDP* and VP4DP* have really complex mixed operand patterns.
   583  	//
   584  	// I think for these we may just have to hand-write a table of which
   585  	// operands each mask applies to.
   586  	inferMask := func(r, w bool) error {
   587  		var masks []int
   588  		var rSizes, wSizes, sizes []vecShape
   589  		allMasks := true
   590  		hasWMask := false
   591  		for i, op := range ops {
   592  			action := op.common().action
   593  			if _, ok := op.(operandMask); ok {
   594  				if action.r && action.w {
   595  					return fmt.Errorf("unexpected rw mask")
   596  				}
   597  				if action.r == r || action.w == w {
   598  					masks = append(masks, i)
   599  				}
   600  				if action.w {
   601  					hasWMask = true
   602  				}
   603  			} else {
   604  				allMasks = false
   605  				if reg, ok := op.(operandVReg); ok {
   606  					if action.r {
   607  						rSizes = append(rSizes, reg.vecShape)
   608  					}
   609  					if action.w {
   610  						wSizes = append(wSizes, reg.vecShape)
   611  					}
   612  				}
   613  			}
   614  		}
   615  		if len(masks) == 0 {
   616  			return nil
   617  		}
   618  
   619  		if r {
   620  			sizes = rSizes
   621  			if len(sizes) == 0 {
   622  				sizes = wSizes
   623  			}
   624  		}
   625  		if w {
   626  			sizes = wSizes
   627  			if len(sizes) == 0 {
   628  				sizes = rSizes
   629  			}
   630  		}
   631  
   632  		if len(sizes) == 0 {
   633  			// If all operands are masks, leave the mask inferrence to the users.
   634  			if allMasks {
   635  				for _, i := range masks {
   636  					m := ops[i].(operandMask)
   637  					m.allMasks = true
   638  					ops[i] = m
   639  				}
   640  				return nil
   641  			}
   642  			return fmt.Errorf("cannot infer mask size: no register operands")
   643  		}
   644  		shape, ok := singular(sizes)
   645  		if !ok {
   646  			if !hasWMask && len(wSizes) == 1 && len(masks) == 1 {
   647  				// This pattern looks like predicate mask, so its shape should align with the
   648  				// output. TODO: verify this is a safe assumption.
   649  				shape = wSizes[0]
   650  			} else {
   651  				return fmt.Errorf("cannot infer mask size: multiple register sizes %v", sizes)
   652  			}
   653  		}
   654  		for _, i := range masks {
   655  			m := ops[i].(operandMask)
   656  			m.vecShape = shape
   657  			ops[i] = m
   658  		}
   659  		return nil
   660  	}
   661  	if err := inferMask(true, false); err != nil {
   662  		return err
   663  	}
   664  	if err := inferMask(false, true); err != nil {
   665  		return err
   666  	}
   667  	return nil
   668  }
   669  
   670  // addOperandsToDef adds "in", "inVariant", and "out" to an instruction Def.
   671  //
   672  // Optional mask input operands are added to the inVariant field if
   673  // variant&instVariantMasked, and omitted otherwise.
   674  func addOperandsToDef(ops []operand, instDB *unify.DefBuilder, variant instVariant) {
   675  	var inVals, inVar, outVals []*unify.Value
   676  	asmPos := 0
   677  	for _, op := range ops {
   678  		var db unify.DefBuilder
   679  		op.addToDef(&db)
   680  		db.Add("asmPos", unify.NewValue(unify.NewStringExact(fmt.Sprint(asmPos))))
   681  
   682  		action := op.common().action
   683  		asmCount := 1 // # of assembly operands; 0 or 1
   684  		if action.r {
   685  			inVal := unify.NewValue(db.Build())
   686  			// If this is an optional mask, put it in the input variant tuple.
   687  			if mask, ok := op.(operandMask); ok && mask.optional {
   688  				if variant&instVariantMasked != 0 {
   689  					inVar = append(inVar, inVal)
   690  				} else {
   691  					// This operand doesn't appear in the assembly at all.
   692  					asmCount = 0
   693  				}
   694  			} else {
   695  				// Just a regular input operand.
   696  				inVals = append(inVals, inVal)
   697  			}
   698  		}
   699  		if action.w {
   700  			outVal := unify.NewValue(db.Build())
   701  			outVals = append(outVals, outVal)
   702  		}
   703  
   704  		asmPos += asmCount
   705  	}
   706  
   707  	instDB.Add("in", unify.NewValue(unify.NewTuple(inVals...)))
   708  	instDB.Add("inVariant", unify.NewValue(unify.NewTuple(inVar...)))
   709  	instDB.Add("out", unify.NewValue(unify.NewTuple(outVals...)))
   710  	memFeatures := checkMem(ops)
   711  	if memFeatures != "noMem" {
   712  		instDB.Add("memFeatures", unify.NewValue(unify.NewStringExact(memFeatures)))
   713  	}
   714  }
   715  
   716  // checkMem checks the shapes of memory operand in the operation and returns the shape.
   717  // Keep this function in sync with [decodeOperand].
   718  func checkMem(ops []operand) string {
   719  	memState := "noMem"
   720  	var mem *operandMem
   721  	memCnt := 0
   722  	for _, op := range ops {
   723  		if m, ok := op.(operandMem); ok {
   724  			mem = &m
   725  			memCnt++
   726  		}
   727  	}
   728  	if mem != nil {
   729  		if mem.unknown {
   730  			memState = "unknown"
   731  		} else if memCnt > 1 {
   732  			memState = "tooManyMem"
   733  		} else {
   734  			// This shape has an indication that [bits] fields has two possible value:
   735  			// 1. The element broadcast width, which is its peer vreg operand's [elemBits] (default val in the parsed XED data)
   736  			// 2. The full vector width, which is its peer vreg operand's [bits] (godefs should be aware of this)
   737  			memState = "hasMem"
   738  		}
   739  	}
   740  	return memState
   741  }
   742  
   743  func instToUVal(inst *xeddata.Inst, ops []operand, addFields map[string]string) []*unify.Value {
   744  	feature, ok := decodeCPUFeature(inst)
   745  	if !ok {
   746  		return nil
   747  	}
   748  
   749  	var vals []*unify.Value
   750  	vals = append(vals, instToUVal1(inst, ops, feature, instVariantNone, addFields))
   751  	if hasOptionalMask(ops) {
   752  		vals = append(vals, instToUVal1(inst, ops, feature, instVariantMasked, addFields))
   753  	}
   754  	return vals
   755  }
   756  
   757  func instToUVal1(inst *xeddata.Inst, ops []operand, feature string, variant instVariant, addFields map[string]string) *unify.Value {
   758  	var db unify.DefBuilder
   759  	db.Add("goarch", unify.NewValue(unify.NewStringExact("amd64")))
   760  	db.Add("asm", unify.NewValue(unify.NewStringExact(inst.Opcode())))
   761  	addOperandsToDef(ops, &db, variant)
   762  	db.Add("cpuFeature", unify.NewValue(unify.NewStringExact(feature)))
   763  	for k, v := range addFields {
   764  		db.Add(k, unify.NewValue(unify.NewStringExact(v)))
   765  	}
   766  
   767  	if strings.Contains(inst.Pattern, "ZEROING=0") {
   768  		// This is an EVEX instruction, but the ".Z" (zero-merging)
   769  		// instruction flag is NOT valid. EVEX.z must be zero.
   770  		//
   771  		// This can mean a few things:
   772  		//
   773  		// - The output of an instruction is a mask, so merging modes don't
   774  		// make any sense. E.g., VCMPPS.
   775  		//
   776  		// - There are no masks involved anywhere. (Maybe MASK=0 is also set
   777  		// in this case?) E.g., VINSERTPS.
   778  		//
   779  		// - The operation inherently performs merging. E.g., VCOMPRESSPS
   780  		// with a mem operand.
   781  		//
   782  		// There may be other reasons.
   783  		db.Add("zeroing", unify.NewValue(unify.NewStringExact("false")))
   784  	}
   785  	pos := unify.Pos{Path: inst.Pos.Path, Line: inst.Pos.Line}
   786  	return unify.NewValuePos(db.Build(), pos)
   787  }
   788  
   789  // decodeCPUFeature returns the CPU feature name required by inst. These match
   790  // the names of the "Has*" feature checks in the simd package.
   791  func decodeCPUFeature(inst *xeddata.Inst) (string, bool) {
   792  	isaSet := inst.ISASet
   793  	if isaSet == "" {
   794  		// Older instructions don't have an ISA set. Use their "extension"
   795  		// instead.
   796  		isaSet = inst.Extension
   797  	}
   798  	// We require AVX512VL to use AVX512 at all, so strip off the vector length
   799  	// suffixes.
   800  	if strings.HasPrefix(isaSet, "AVX512") {
   801  		isaSet = isaSetVL.ReplaceAllLiteralString(isaSet, "")
   802  	}
   803  
   804  	feat, ok := cpuFeatureMap[isaSet]
   805  	if !ok {
   806  		imap := unknownFeatures[isaSet]
   807  		if imap == nil {
   808  			imap = make(map[string]struct{})
   809  			unknownFeatures[isaSet] = imap
   810  		}
   811  		imap[inst.Opcode()] = struct{}{}
   812  		return "", false
   813  	}
   814  	if feat == "ignore" {
   815  		return "", false
   816  	}
   817  	return feat, true
   818  }
   819  
   820  var isaSetVL = regexp.MustCompile("_(128N?|256N?|512)$")
   821  
   822  // cpuFeatureMap maps from XED's "ISA_SET" (or "EXTENSION") to a CPU feature
   823  // name to expose in the SIMD feature check API.
   824  //
   825  // See XED's datafiles/*/cpuid.xed.txt for how ISA set names map to CPUID flags.
   826  var cpuFeatureMap = map[string]string{
   827  	"AVX":      "AVX",
   828  	"AVX_VNNI": "AVXVNNI",
   829  	"AVX2":     "AVX2",
   830  	"AVXAES":   "AVXAES",
   831  	"SHA":      "SHA",
   832  	"FMA":      "FMA",
   833  	"VAES":     "VAES",
   834  
   835  	// AVX-512 foundational features. We combine all of these into one "AVX512" feature.
   836  	"AVX512F":  "AVX512",
   837  	"AVX512BW": "AVX512",
   838  	"AVX512CD": "AVX512",
   839  	"AVX512DQ": "AVX512",
   840  	// AVX512VL doesn't appear as its own ISASet; instead, the CPUID flag is
   841  	// required by the *_128 and *_256 ISASets. We fold it into "AVX512" anyway.
   842  
   843  	// AVX-512 extension features
   844  	"AVX512_BITALG":     "AVX512BITALG",
   845  	"AVX512_GFNI":       "AVX512GFNI",
   846  	"AVX512_VBMI":       "AVX512VBMI",
   847  	"AVX512_VBMI2":      "AVX512VBMI2",
   848  	"AVX512_VNNI":       "AVX512VNNI",
   849  	"AVX512_VPOPCNTDQ":  "AVX512VPOPCNTDQ",
   850  	"AVX512_VAES":       "AVX512VAES",
   851  	"AVX512_VPCLMULQDQ": "AVX512VPCLMULQDQ",
   852  
   853  	// AVX 10.2 (not yet supported)
   854  	"AVX10_2_RC": "ignore",
   855  }
   856  
   857  func init() {
   858  	// TODO: In general, Intel doesn't make any guarantees about what flags are
   859  	// set, so this means our feature checks need to ensure these, just to be
   860  	// sure.
   861  	var features = map[string]featureInfo{
   862  		"AVX2":   {Implies: []string{"AVX"}},
   863  		"AVX512": {Implies: []string{"AVX2"}},
   864  
   865  		"AVXAES": {Virtual: true, Implies: []string{"AVX", "AES"}},
   866  		"FMA":    {Implies: []string{"AVX"}},
   867  		"VAES":   {Implies: []string{"AVX"}},
   868  
   869  		// AVX-512 subfeatures.
   870  		"AVX512BITALG":    {Implies: []string{"AVX512"}},
   871  		"AVX512GFNI":      {Implies: []string{"AVX512"}},
   872  		"AVX512VBMI":      {Implies: []string{"AVX512"}},
   873  		"AVX512VBMI2":     {Implies: []string{"AVX512"}},
   874  		"AVX512VNNI":      {Implies: []string{"AVX512"}},
   875  		"AVX512VPOPCNTDQ": {Implies: []string{"AVX512"}},
   876  		"AVX512VAES":      {Implies: []string{"AVX512"}},
   877  
   878  		// AVX-VNNI and AVX-IFMA are "backports" of the AVX512-VNNI/IFMA
   879  		// instructions to VEX encoding, limited to 256 bit vectors. They're
   880  		// intended for lower end CPUs that want to support VNNI/IFMA without
   881  		// supporting AVX-512. As such, they're built on AVX2's VEX encoding.
   882  		"AVXVNNI": {Implies: []string{"AVX2"}},
   883  		"AVXIFMA": {Implies: []string{"AVX2"}},
   884  	}
   885  	registerFeatureInfo("amd64", goarchFeatures{
   886  		featureVar: "X86",
   887  		features:   features,
   888  	})
   889  }
   890  
   891  var unknownFeatures = map[string]map[string]struct{}{}
   892  
   893  // hasOptionalMask returns whether there is an optional mask operand in ops.
   894  func hasOptionalMask(ops []operand) bool {
   895  	for _, op := range ops {
   896  		if op, ok := op.(operandMask); ok && op.optional {
   897  			return true
   898  		}
   899  	}
   900  	return false
   901  }
   902  
   903  func singular[T comparable](xs []T) (T, bool) {
   904  	if len(xs) == 0 {
   905  		return *new(T), false
   906  	}
   907  	for _, x := range xs[1:] {
   908  		if x != xs[0] {
   909  			return *new(T), false
   910  		}
   911  	}
   912  	return xs[0], true
   913  }
   914  
   915  type fixedReg struct {
   916  	class int
   917  	name  string
   918  	width int
   919  }
   920  
   921  var fixedRegMap = map[string]fixedReg{
   922  	"XED_REG_XMM0": {VREG_CLASS, "x0", 128},
   923  }
   924  
   925  // decodeReg returns class (NOT_REG_CLASS, VREG_CLASS, GREG_CLASS, VREG_CLASS_FIXED,
   926  // GREG_CLASS_FIXED), width in bits and reg name(if fixed).
   927  // If the operand cannot be decided as a register, then the clas is NOT_REG_CLASS.
   928  func decodeReg(op *xeddata.Operand) (class, width int, name string) {
   929  	// op.Width tells us the total width, e.g.,:
   930  	//
   931  	//    dq => 128 bits (XMM)
   932  	//    qq => 256 bits (YMM)
   933  	//    mskw => K
   934  	//    z[iuf?](8|16|32|...) => 512 bits (ZMM)
   935  	//
   936  	// But the encoding is really weird and it's not clear if these *always*
   937  	// mean XMM/YMM/ZMM or if other irregular things can use these large widths.
   938  	// Hence, we dig into the register sets themselves.
   939  
   940  	if !strings.HasPrefix(op.NameLHS(), "REG") {
   941  		return NOT_REG_CLASS, 0, ""
   942  	}
   943  	// TODO: We shouldn't be relying on the macro naming conventions. We should
   944  	// use all-dec-patterns.txt, but xeddata doesn't support that table right now.
   945  	rhs := op.NameRHS()
   946  	if !strings.HasSuffix(rhs, "()") {
   947  		if fixedReg, ok := fixedRegMap[rhs]; ok {
   948  			return fixedReg.class, fixedReg.width, fixedReg.name
   949  		}
   950  		return NOT_REG_CLASS, 0, ""
   951  	}
   952  	switch {
   953  	case strings.HasPrefix(rhs, "XMM_"):
   954  		return VREG_CLASS, 128, ""
   955  	case strings.HasPrefix(rhs, "YMM_"):
   956  		return VREG_CLASS, 256, ""
   957  	case strings.HasPrefix(rhs, "ZMM_"):
   958  		return VREG_CLASS, 512, ""
   959  	case strings.HasPrefix(rhs, "GPR64_"), strings.HasPrefix(rhs, "VGPR64_"):
   960  		return GREG_CLASS, 64, ""
   961  	case strings.HasPrefix(rhs, "GPR32_"), strings.HasPrefix(rhs, "VGPR32_"):
   962  		return GREG_CLASS, 32, ""
   963  	}
   964  	return NOT_REG_CLASS, 0, ""
   965  }
   966  
   967  var xtypeRe = regexp.MustCompile(`^([iuf])([0-9]+)$`)
   968  
   969  // scalarBaseType describes the base type of a scalar element. This is a Go
   970  // type, but without the bit width suffix (with the exception of
   971  // scalarBaseIntOrUint).
   972  type scalarBaseType int
   973  
   974  const (
   975  	scalarBaseInt scalarBaseType = iota
   976  	scalarBaseUint
   977  	scalarBaseIntOrUint // Signed or unsigned is unspecified
   978  	scalarBaseFloat
   979  	scalarBaseComplex
   980  	scalarBaseBFloat
   981  	scalarBaseHFloat
   982  )
   983  
   984  func (s scalarBaseType) regex() string {
   985  	switch s {
   986  	case scalarBaseInt:
   987  		return "int"
   988  	case scalarBaseUint:
   989  		return "uint"
   990  	case scalarBaseIntOrUint:
   991  		return "int|uint"
   992  	case scalarBaseFloat:
   993  		return "float"
   994  	case scalarBaseComplex:
   995  		return "complex"
   996  	case scalarBaseBFloat:
   997  		return "BFloat"
   998  	case scalarBaseHFloat:
   999  		return "HFloat"
  1000  	}
  1001  	panic(fmt.Sprintf("unknown scalar base type %d", s))
  1002  }
  1003  
  1004  func decodeType(op *xeddata.Operand) (base scalarBaseType, bits int, ok bool) {
  1005  	// The xtype tells you the element type. i8, i16, i32, i64, f32, etc.
  1006  	//
  1007  	// TODO: Things like AVX2 VPAND have an xtype of u256 because they're
  1008  	// element-width agnostic. Do I map that to all widths, or just omit the
  1009  	// element width and let unification flesh it out? There's no u512
  1010  	// (presumably those are all masked, so elem width matters). These are all
  1011  	// Category: LOGICAL, so maybe we could use that info?
  1012  
  1013  	// Handle some weird ones.
  1014  	switch op.Xtype {
  1015  	// 8-bit float formats as defined by Open Compute Project "OCP 8-bit
  1016  	// Floating Point Specification (OFP8)".
  1017  	case "bf8": // E5M2 float
  1018  		return scalarBaseBFloat, 8, true
  1019  	case "hf8": // E4M3 float
  1020  		return scalarBaseHFloat, 8, true
  1021  	case "bf16": // bfloat16 float
  1022  		return scalarBaseBFloat, 16, true
  1023  	case "2f16":
  1024  		// Complex consisting of 2 float16s. Doesn't exist in Go, but we can say
  1025  		// what it would be.
  1026  		return scalarBaseComplex, 32, true
  1027  	case "2i8", "2I8":
  1028  		// These just use the lower INT8 in each 16 bit field.
  1029  		// As far as I can tell, "2I8" is a typo.
  1030  		return scalarBaseInt, 8, true
  1031  	case "2u16", "2U16":
  1032  		// some VPDP* has it
  1033  		// TODO: does "z" means it has zeroing?
  1034  		return scalarBaseUint, 16, true
  1035  	case "2i16", "2I16":
  1036  		// some VPDP* has it
  1037  		return scalarBaseInt, 16, true
  1038  	case "4u8", "4U8":
  1039  		// some VPDP* has it
  1040  		return scalarBaseUint, 8, true
  1041  	case "4i8", "4I8":
  1042  		// some VPDP* has it
  1043  		return scalarBaseInt, 8, true
  1044  	}
  1045  
  1046  	// The rest follow a simple pattern.
  1047  	m := xtypeRe.FindStringSubmatch(op.Xtype)
  1048  	if m == nil {
  1049  		// TODO: Report unrecognized xtype
  1050  		return 0, 0, false
  1051  	}
  1052  	bits, _ = strconv.Atoi(m[2])
  1053  	switch m[1] {
  1054  	case "i", "u":
  1055  		// XED is rather inconsistent about what's signed, unsigned, or doesn't
  1056  		// matter, so merge them together and let the Go definitions narrow as
  1057  		// appropriate. Maybe there's a better way to do this.
  1058  		return scalarBaseIntOrUint, bits, true
  1059  	case "f":
  1060  		return scalarBaseFloat, bits, true
  1061  	default:
  1062  		panic("unreachable")
  1063  	}
  1064  }
  1065  

View as plain text