[pull] master from golang:master

src/bytes/bytes_test.go

@@ -762,7 +762,7 @@ func BenchmarkEqual(b *testing.B) {
 		}
 	})
 
-	sizes := []int{1, 6, 9, 15, 16, 20, 32, 4 << 10, 4 << 20, 64 << 20}
+	sizes := []int{1, 6, 9, 15, 16, 20, 32, 65, 96, 100, 127, 4 << 10, 4 << 20, 64 << 20}
 
 	b.Run("same", func(b *testing.B) {
 		benchBytes(b, sizes, bmEqual(func(a, b []byte) bool { return Equal(a, a) }))

src/cmd/compile/internal/noder/reader.go

@@ -11,6 +11,7 @@ import (
 	"internal/buildcfg"
 	"internal/pkgbits"
 	"path/filepath"
+	"slices"
 	"strings"
 
 	"cmd/compile/internal/base"
@@ -3542,6 +3543,51 @@ func (r *reader) pkgInitOrder(target *ir.Package) {
 	typecheck.DeclFunc(fn)
 	r.curfn = fn
 
+	var varInitFns []*ir.Func
+	if len(initOrder) <= maxInitStatements {
+		fn.Body = r.doPkgInitOrder(initOrder)
+	} else {
+		varInitFns = r.splitLargeInitOrder(initOrder)
+		calls := make([]ir.Node, len(varInitFns))
+		for i, varInitFn := range varInitFns {
+			ir.WithFunc(fn, func() {
+				calls[i] = typecheck.Call(varInitFn.Pos(), varInitFn.Nname, nil, false)
+			})
+		}
+		fn.Body = calls
+	}
+
+	typecheck.FinishFuncBody()
+	r.curfn = nil
+	r.locals = nil
+
+	// Outline (if legal/profitable) global map inits.
+	staticinit.OutlineMapInits(fn)
+	for _, varInitFn := range varInitFns {
+		staticinit.OutlineMapInits(varInitFn)
+	}
+
+	target.Inits = append(target.Inits, fn)
+	target.Inits = append(target.Inits, varInitFns...)
+}
+
+const maxInitStatements = 1000
+
+func (r *reader) generateVarInitFunc(body []ir.Node) *ir.Func {
+	fn := staticinit.GenerateVarInitFunc()
+	typecheck.DeclFunc(fn)
+
+	old := r.curfn
+	r.curfn = fn
+	fn.Body = r.doPkgInitOrder(body)
+	r.curfn = old
+
+	typecheck.FinishFuncBody()
+
+	return fn
+}
+
+func (r *reader) doPkgInitOrder(initOrder []ir.Node) []ir.Node {
 	for i := range initOrder {
 		lhs := make([]ir.Node, r.Len())
 		for j := range lhs {
@@ -3563,17 +3609,15 @@ func (r *reader) pkgInitOrder(target *ir.Package) {
 
 		initOrder[i] = as
 	}
+	return initOrder
+}
 
-	fn.Body = initOrder
-
-	typecheck.FinishFuncBody()
-	r.curfn = nil
-	r.locals = nil
-
-	// Outline (if legal/profitable) global map inits.
-	staticinit.OutlineMapInits(fn)
-
-	target.Inits = append(target.Inits, fn)
+func (r *reader) splitLargeInitOrder(initOrder []ir.Node) []*ir.Func {
+	var initFuncs []*ir.Func
+	for chunk := range slices.Chunk(initOrder, maxInitStatements) {
+		initFuncs = append(initFuncs, r.generateVarInitFunc(chunk))
+	}
+	return initFuncs
 }
 
 func (r *reader) pkgDecls(target *ir.Package) {

src/cmd/compile/internal/pkginit/init.go

@@ -87,10 +87,7 @@ func MakeTask() {
 
 	// Record user init functions.
 	for _, fn := range typecheck.Target.Inits {
-		if fn.Sym().Name == "init" {
-			// Synthetic init function for initialization of package-scope
-			// variables. We can use staticinit to optimize away static
-			// assignments.
+		if staticinit.CanOptimize(fn) {
 			s := staticinit.Schedule{
 				Plans: make(map[ir.Node]*staticinit.Plan),
 				Temps: make(map[ir.Node]*ir.Name),

src/cmd/compile/internal/ssa/_gen/generic.rules

@@ -74,6 +74,13 @@
 (PopCount32 (Const32 [c])) && config.PtrSize == 4 => (Const32 [int32(bits.OnesCount32(uint32(c)))])
 (PopCount16 (Const16 [c])) && config.PtrSize == 4 => (Const32 [int32(bits.OnesCount16(uint16(c)))])
 (PopCount8  (Const8  [c])) && config.PtrSize == 4 => (Const32 [int32(bits.OnesCount8(uint8(c)))])
+(Bswap64 (Const64 [c])) => (Const64 [int64(bits.ReverseBytes64(uint64(c)))])
+(Bswap32 (Const32 [c])) => (Const32 [int32(bits.ReverseBytes32(uint32(c)))])
+(Bswap16 (Const16 [c])) => (Const16 [int16(bits.ReverseBytes16(uint16(c)))])
+(BitRev64 (Const64 [c])) => (Const64 [int64(bits.Reverse64(uint64(c)))])
+(BitRev32 (Const32 [c])) => (Const32 [int32(bits.Reverse32(uint32(c)))])
+(BitRev16 (Const16 [c])) => (Const16 [int16(bits.Reverse16(uint16(c)))])
+(BitRev8  (Const8  [c])) => (Const8  [int8(bits.Reverse8(uint8(c)))])
 (Add64carry (Const64 <t> [x]) (Const64 [y]) (Const64 [c])) && c >= 0 && c <= 1 => (MakeTuple (Const64 <t> [bitsAdd64(x, y, c).sum]) (Const64 <t> [bitsAdd64(x, y, c).carry]))
 
 (Trunc16to8  (ZeroExt8to16  x)) => x
@@ -144,6 +151,13 @@
 (Mul32uover (Const32 [c]) (Const32 [d])) => (MakeTuple (Const32 <typ.UInt32> [bitsMulU32(c, d).lo]) (ConstBool <typ.Bool> [bitsMulU32(c,d).hi != 0]))
 (Mul64uover (Const64 [c]) (Const64 [d])) => (MakeTuple (Const64 <typ.UInt64> [bitsMulU64(c, d).lo]) (ConstBool <typ.Bool> [bitsMulU64(c,d).hi != 0]))
 
+// bits.Mul64(x, 1<<s) for 0 < s < 64.
+//   hi:lo = (x >> (64-s), x << s)
+(Mul64uhilo x (Const64 [c])) && c > 0 && isPowerOfTwo(uint64(c)) =>
+  (MakeTuple
+    (Rsh64Ux64 <typ.UInt64> x (Const64 <typ.UInt64> [64 - log64u(uint64(c))]))
+    (Lsh64x64  <typ.UInt64> x (Const64 <typ.UInt64> [log64u(uint64(c))])))
+
 (And8   (Const8 [c])   (Const8 [d]))   => (Const8  [c&d])
 (And16  (Const16 [c])  (Const16 [d]))  => (Const16 [c&d])
 (And32  (Const32 [c])  (Const32 [d]))  => (Const32 [c&d])
@@ -260,12 +274,12 @@
 (IsInBounds (ZeroExt8to64  (Rsh8Ux64  _ (Const64 [c]))) (Const64 [d])) && 0 < c && c <  8 && 1<<uint( 8-c)-1 < d => (ConstBool [true])
 (IsInBounds (ZeroExt8to32  (Rsh8Ux64  _ (Const64 [c]))) (Const32 [d])) && 0 < c && c <  8 && 1<<uint( 8-c)-1 < d => (ConstBool [true])
 (IsInBounds (ZeroExt8to16  (Rsh8Ux64  _ (Const64 [c]))) (Const16 [d])) && 0 < c && c <  8 && 1<<uint( 8-c)-1 < d => (ConstBool [true])
-(IsInBounds                (Rsh8Ux64  _ (Const64 [c]))  (Const64 [d])) && 0 < c && c <  8 && 1<<uint( 8-c)-1 < d => (ConstBool [true])
+(IsInBounds                (Rsh8Ux64  _ (Const64 [c]))  (Const8  [d])) && 0 < c && c <  8 && 1<<uint( 8-c)-1 < d => (ConstBool [true])
 (IsInBounds (ZeroExt16to64 (Rsh16Ux64 _ (Const64 [c]))) (Const64 [d])) && 0 < c && c < 16 && 1<<uint(16-c)-1 < d => (ConstBool [true])
-(IsInBounds (ZeroExt16to32 (Rsh16Ux64 _ (Const64 [c]))) (Const64 [d])) && 0 < c && c < 16 && 1<<uint(16-c)-1 < d => (ConstBool [true])
-(IsInBounds                (Rsh16Ux64 _ (Const64 [c]))  (Const64 [d])) && 0 < c && c < 16 && 1<<uint(16-c)-1 < d => (ConstBool [true])
+(IsInBounds (ZeroExt16to32 (Rsh16Ux64 _ (Const64 [c]))) (Const32 [d])) && 0 < c && c < 16 && 1<<uint(16-c)-1 < d => (ConstBool [true])
+(IsInBounds                (Rsh16Ux64 _ (Const64 [c]))  (Const16 [d])) && 0 < c && c < 16 && 1<<uint(16-c)-1 < d => (ConstBool [true])
 (IsInBounds (ZeroExt32to64 (Rsh32Ux64 _ (Const64 [c]))) (Const64 [d])) && 0 < c && c < 32 && 1<<uint(32-c)-1 < d => (ConstBool [true])
-(IsInBounds                (Rsh32Ux64 _ (Const64 [c]))  (Const64 [d])) && 0 < c && c < 32 && 1<<uint(32-c)-1 < d => (ConstBool [true])
+(IsInBounds                (Rsh32Ux64 _ (Const64 [c]))  (Const32 [d])) && 0 < c && c < 32 && 1<<uint(32-c)-1 < d => (ConstBool [true])
 (IsInBounds                (Rsh64Ux64 _ (Const64 [c]))  (Const64 [d])) && 0 < c && c < 64 && 1<<uint(64-c)-1 < d => (ConstBool [true])
 
 (IsSliceInBounds x x) => (ConstBool [true])
@@ -2408,3 +2422,11 @@
 
 // Canonicalize sext(int(bool)) => zext(int(bool))
 (SignExt8to(64|32|16) cvt:(CvtBoolToUint8 bool)) => (ZeroExt8to(64|32|16) cvt)
+
+// int(bool)^1 => int(!bool)
+(Xor8                                (CvtBoolToUint8        bool)  (Const8 [1]))          && invertibleBool(bool.Op) =>                       (CvtBoolToUint8        (Not <bool.Type> bool))
+(Xor(64|32|16) (ZeroExt8to(64|32|16) (CvtBoolToUint8 <cvtT> bool)) (Const(64|32|16) [1])) && invertibleBool(bool.Op) => (ZeroExt8to(64|32|16) (CvtBoolToUint8 <cvtT> (Not <bool.Type> bool)))
+
+// int(!bool)^c => int(bool)^(c^1)
+(Xor8                                (CvtBoolToUint8 <cvtT> (Not bool))  (Const8          <constT> [c])) && c != 1 => (Xor8                                (CvtBoolToUint8 <cvtT> bool)  (Const8          <constT> [c^1]))
+(Xor(64|32|16) (ZeroExt8to(64|32|16) (CvtBoolToUint8 <cvtT> (Not bool))) (Const(64|32|16) <constT> [c])) && c != 1 => (Xor(64|32|16) (ZeroExt8to(64|32|16) (CvtBoolToUint8 <cvtT> bool)) (Const(64|32|16) <constT> [c^1]))

src/cmd/compile/internal/ssa/memcombine.go

@@ -90,41 +90,109 @@ func memcombineLoads(f *Func) {
 // idx may be nil, in which case it is treated as 0.
 type BaseAddress struct {
 	ptr *Value
-	idx *Value
+	idx Index
+}
+
+// Index represents an address index in the form exp<<shift.
+//
+// The shift is typically introduced by slice indexing (log2(element size)),
+// but may also originate from shifts in the source expression.
+type Index struct {
+	exp   *Value
+	shift int64
+}
+
+func getConst(v *Value) (int64, bool) {
+	if v.Op == OpConst32 || v.Op == OpConst64 {
+		return v.AuxInt, true
+	}
+	return 0, false
+}
+
+func peelAdd(v *Value) (exp *Value, imm int64) {
+	if v == nil {
+		return nil, 0
+	}
+
+	if v.Op == OpAdd32 || v.Op == OpAdd64 {
+		if imm, ok := getConst(v.Args[0]); ok {
+			return v.Args[1], imm
+		}
+
+		if imm, ok := getConst(v.Args[1]); ok {
+			return v.Args[0], imm
+		}
+	}
+
+	return v, 0
+}
+
+func peelShift(v *Value) (exp *Value, shift int64) {
+	if v == nil {
+		return nil, 0
+	}
+
+	if v.Op == OpLsh64x64 || v.Op == OpLsh32x64 || v.Op == OpLsh16x64 {
+		if imm, ok := getConst(v.Args[1]); ok {
+			return v.Args[0], imm
+		}
+	}
+	return v, 0
 }
 
 // splitPtr returns the base address of ptr and any
 // constant offset from that base.
 // BaseAddress{ptr,nil},0 is always a valid result, but splitPtr
 // tries to peel away as many constants into off as possible.
 func splitPtr(ptr *Value) (BaseAddress, int64) {
-	var idx *Value
+	var idx Index
 	var off int64
 	for {
 		if ptr.Op == OpOffPtr {
 			off += ptr.AuxInt
 			ptr = ptr.Args[0]
-		} else if ptr.Op == OpAddPtr {
-			if idx != nil {
+			continue
+		}
+
+		if ptr.Op == OpAddPtr {
+			if idx.exp != nil {
 				// We have two or more indexing values.
 				// Pick the first one we found.
-				return BaseAddress{ptr: ptr, idx: idx}, off
-			}
-			idx = ptr.Args[1]
-			if idx.Op == OpAdd32 || idx.Op == OpAdd64 {
-				if idx.Args[0].Op == OpConst32 || idx.Args[0].Op == OpConst64 {
-					off += idx.Args[0].AuxInt
-					idx = idx.Args[1]
-				} else if idx.Args[1].Op == OpConst32 || idx.Args[1].Op == OpConst64 {
-					off += idx.Args[1].AuxInt
-					idx = idx.Args[0]
-				}
+				break
 			}
+
+			// Common slice indexing patterns:
+			//
+			// exp
+			// exp + offset
+			// (exp << shift) + offset
+			// (exp+imm)<<shift + offset
+			//
+			// where shift is typically log2(element size).
+
+			idx.exp = ptr.Args[1]
 			ptr = ptr.Args[0]
-		} else {
-			return BaseAddress{ptr: ptr, idx: idx}, off
+
+			// Peel offset
+			var offset int64
+			idx.exp, offset = peelAdd(idx.exp)
+			off += offset
+
+			// Peel shift
+			idx.exp, idx.shift = peelShift(idx.exp)
+
+			// Peel imm
+			var imm int64
+			idx.exp, imm = peelAdd(idx.exp)
+
+			off += imm << idx.shift
+			continue
 		}
+
+		break
 	}
+
+	return BaseAddress{ptr: ptr, idx: idx}, off
 }
 
 func combineLoads(root *Value, n int64) bool {
@@ -203,11 +271,7 @@ func combineLoads(root *Value, n int64) bool {
 		}
 		shift := int64(0)
 		if v.Op == shiftOp {
-			if v.Args[1].Op != OpConst64 {
-				return false
-			}
-			shift = v.Args[1].AuxInt
-			v = v.Args[0]
+			v, shift = peelShift(v)
 			if v.Uses != 1 {
 				return false
 			}

src/cmd/compile/internal/ssa/rewrite.go

@@ -2825,17 +2825,9 @@ func bool2int(x bool) int {
 func rewriteCondSelectIntoMath(config *Config, op Op, constant int64) bool {
 	switch config.arch {
 	case "amd64":
-		if constant == 1 {
-			return true
-		}
-		switch op {
-		case OpAdd64, OpAdd32, OpAdd16, OpAdd8:
-			switch constant {
-			case 2, 4, 8:
-				// Implemented with LEA a + b * displacement form
-				return true
-			}
-		}
+		// constant=1 becomes zext, add 2/4/8 becomes lea, rest becomes shl.
+		// shl has asymmetric latency (1:3 vs 2:2) but performs better in accumulation chains.
+		return isPowerOfTwo(uint64(constant))
 	case "arm64":
 		switch op {
 		case OpAdd64, OpAdd32, OpAdd16, OpAdd8:
@@ -2891,3 +2883,18 @@ func modularMultiplicativeInverse(x uint64) (y uint64) {
 	y *= 2 - x*y // 96 bits; good enough
 	return
 }
+
+func invertibleBool(op Op) bool {
+	switch op {
+	case OpLess64, OpLess32, OpLess16, OpLess8,
+		OpLeq64, OpLeq32, OpLeq16, OpLeq8,
+		OpLess64U, OpLess32U, OpLess16U, OpLess8U,
+		OpLeq64U, OpLeq32U, OpLeq16U, OpLeq8U,
+		OpEq64, OpEq32, OpEq16, OpEq8,
+		OpNeq64, OpNeq32, OpNeq16, OpNeq8,
+		OpNot:
+		return true
+	default:
+		return false
+	}
+}