OP#792 initial commit after fork

internal/grid/README.md

@@ -0,0 +1,251 @@
+# MinIO Grid
+
+The MinIO Grid is a package that provides two-way communication between servers.
+It uses a single two-way connection to send and receive messages between servers.
+
+It includes built in muxing of concurrent requests as well as congestion handling for streams.
+
+Requests can be "Single Payload" or "Streamed".
+
+Use the MinIO Grid for:
+
+* Small, frequent requests with low latency requirements.
+* Long-running requests with small/medium payloads.
+
+Do *not* use the MinIO Grid for:
+
+* Large payloads.
+
+Only a single connection is ever made between two servers.
+Likely this means that this connection will not be able to saturate network bandwidth.
+Therefore, using this for large payloads will likely be slower than using a separate connection,
+and other connections will be blocked while the large payload is being sent.
+
+## Handlers & Routes
+
+Handlers have a predefined Handler ID.
+In addition, there can be several *static* subroutes used to differentiate between different handlers of the same ID.
+A subroute on a client must match a subroute on the server. So routes cannot be used for dynamic routing, unlike HTTP.
+
+Handlers should remain backwards compatible. If a breaking API change is required, a new handler ID should be created.
+
+## Setup & Configuration
+
+A **Manager** is used to manage all incoming and outgoing connections to a server.
+
+On startup all remote servers must be specified.
+From that individual connections will be spawned to each remote server,
+or incoming requests will be hooked up to the appropriate connection.
+
+To get a connection to a specific server, use `Manager.Connection(host)` to get a connection to the specified host.
+From this connection individual requests can be made.
+
+Each handler, with optional subroutes can be registered with the manager using
+`Manager.RegisterXHandler(handlerID, handler, subroutes...)`.
+
+A `Handler()` function provides an HTTP handler, which should be hooked up to the appropriate route on the server.
+
+On startup, the manager will start connecting to remotes and also starts listening for incoming connections.
+Until a connection is established, all outgoing requests will return `ErrDisconnected`.
+
+# Usage
+
+## Single Payload Requests
+
+Single payload requests are requests and responses that are sent in a single message.
+In essence, they are `[]byte` -> `[]byte, error` functions.
+
+It is not possible to return *both* an error and a response.
+
+Handlers are registered on the manager using `(*Manager).RegisterSingleHandler(id HandlerID, h SingleHandlerFn, subroute ...string)`.
+
+The server handler function has this signature: `type SingleHandlerFn func(payload []byte) ([]byte, *RemoteErr)`.
+
+Sample handler:
+```go
+    handler :=  func(payload []byte) ([]byte, *grid.RemoteErr) {
+        // Do something with payload
+        return []byte("response"), nil
+    }
+
+    err := manager.RegisterSingleHandler(grid.HandlerDiskInfo, handler)
+```
+
+Sample call:
+```go
+    // Get a connection to the remote host
+    conn := manager.Connection(host)
+
+    payload := []byte("request")
+    response, err := conn.SingleRequest(ctx, grid.HandlerDiskInfo, payload)
+```
+
+If the error type is `*RemoteErr`, then the error was returned by the remote server. Otherwise it is a local error.
+
+Context timeouts are propagated, and a default timeout of 1 minute is added if none is specified.
+
+There is no cancellation propagation for single payload requests.
+When the context is canceled, the request will return at once with an appropriate error.
+However, the remote call will not see the cancellation - as can be seen from the 'missing' context on the handler.
+The result will be discarded.
+
+### Typed handlers
+
+Typed handlers are handlers that have a specific type for the request and response payloads.
+These must provide `msgp` serialization and deserialization.
+
+In the examples we use a `MSS` type, which is a `map[string]string` that is `msgp` serializable.
+
+```go
+    handler := func(request *grid.MSS) (*grid.MSS, *grid.RemoteErr) {
+        fmt.Println("Got request with field", request["myfield"])
+        // Do something with payload
+        return NewMSSWith(map[string]string{"result": "ok"}), nil
+    }
+
+    // Create a typed handler.
+    // Due to current generics limitations, a constructor of the empty type must be provided.
+    instance := grid.NewSingleHandler[*grid.MSS, *grid.MSS](h, grid.NewMSS, grid.NewMSS)
+
+    // Register the handler on the manager
+    instance.Register(manager, handler)
+
+    // The typed instance is also used for calls
+    conn := manager.Connection("host")
+    resp, err := instance.Call(ctx, conn, grid.NewMSSWith(map[string]string{"myfield": "myvalue"}))
+    if err == nil {
+        fmt.Println("Got response with field", resp["result"])
+    }
+```
+
+The wrapper will handle all serialization and de-serialization of the request and response,
+and furthermore provides reuse of the structs used for the request and response.
+
+Note that Responses sent for serialization are automatically reused for similar requests.
+If the response contains shared data it will cause issues, since each unique response is reused.
+To disable this behavior, use `(SingleHandler).WithSharedResponse()` to disable it.
+
+## Streaming Requests
+
+Streams consists of an initial request with payload and allows for full two-way communication between the client and server.
+
+The handler function has this signature.
+
+Sample handler:
+```go
+    handler :=  func(ctx context.Context, payload []byte, in <-chan []byte, out chan<- []byte) *RemoteErr {
+        fmt.Println("Got request with initial payload", p, "from", GetCaller(ctx context.Context))
+        fmt.Println("Subroute:", GetSubroute(ctx))
+        for {
+            select {
+            case <-ctx.Done():
+                return nil
+            case req, ok := <-in:
+                if !ok {
+                    break
+                }
+                // Do something with payload
+                out <- []byte("response")
+
+                // Return the request for reuse
+                grid.PutByteBuffer(req)
+            }
+        }
+        // out is closed by the caller and should never be closed by the handler.
+        return nil
+    }
+
+    err := manager.RegisterStreamingHandler(grid.HandlerDiskInfo, StreamHandler{
+        Handle: handler,
+        Subroute: "asubroute",
+        OutCapacity: 1,
+        InCapacity: 1,
+    })
+```
+
+Sample call:
+```go
+    // Get a connection to the remote host
+    conn := manager.Connection(host).Subroute("asubroute")
+
+    payload := []byte("request")
+    stream, err := conn.NewStream(ctx, grid.HandlerDiskInfo, payload)
+	if err != nil {
+        return err
+    }
+    // Read results from the stream
+    err = stream.Results(func(result []byte) error {
+        fmt.Println("Got result", string(result))
+
+        // Return the response for reuse
+        grid.PutByteBuffer(result)
+        return nil
+    })
+```
+
+Context cancellation and timeouts are propagated to the handler.
+The client does not wait for the remote handler to finish before returning.
+Returning any error will also cancel the stream remotely.
+
+CAREFUL: When utilizing two-way communication, it is important to ensure that the remote handler is not blocked on a send.
+If the remote handler is blocked on a send, and the client is trying to send without the remote receiving,
+the operation would become deadlocked if the channels are full.
+
+### Typed handlers
+
+Typed handlers are handlers that have a specific type for the request and response payloads.
+
+```go
+    // Create a typed handler.
+    handler := func(ctx context.Context, p *Payload, in <-chan *Req, out chan<- *Resp) *RemoteErr {
+        fmt.Println("Got request with initial payload", p, "from", GetCaller(ctx context.Context))
+		fmt.Println("Subroute:", GetSubroute(ctx))
+        for {
+            select {
+            case <-ctx.Done():
+                return nil
+            case req, ok := <-in:
+                if !ok {
+                    break
+                }
+                fmt.Println("Got request", in)
+                // Do something with payload
+                out <- Resp{"response"}
+            }
+            // out is closed by the caller and should never be closed by the handler.
+            return nil
+    }
+
+    // Create a typed handler.
+    // Due to current generics limitations, a constructor of the empty type must be provided.
+    instance := grid.NewStream[*Payload, *Req, *Resp](h, newPayload, newReq, newResp)
+
+    // Tweakable options
+    instance.WithPayload = true // default true when newPayload != nil
+    instance.OutCapacity = 1    // default
+    instance.InCapacity = 1     // default true when newReq != nil
+
+    // Register the handler on the manager
+    instance.Register(manager, handler, "asubroute")
+
+    // The typed instance is also used for calls
+    conn := manager.Connection("host").Subroute("asubroute")
+    stream, err := instance.Call(ctx, conn, &Payload{"request payload"})
+    if err != nil { ... }
+
+    // Read results from the stream
+    err = stream.Results(func(resp *Resp) error {
+        fmt.Println("Got result", resp)
+        // Return the response for reuse
+		instance.PutResponse(resp)
+        return nil
+    })
+```
+
+There are handlers for requests with:
+ * No input stream: `RegisterNoInput`.
+ * No initial payload: `RegisterNoPayload`.
+
+Note that Responses sent for serialization are automatically reused for similar requests.
+If the response contains shared data it will cause issues, since each unique response is reused.
+To disable this behavior, use `(StreamTypeHandler).WithSharedResponse()` to disable it.

internal/grid/benchmark_test.go

@@ -0,0 +1,567 @@
+// Copyright (c) 2015-2023 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package grid
+
+import (
+	"context"
+	"fmt"
+	"math/rand"
+	"runtime"
+	"strconv"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/minio/minio/internal/logger/target/testlogger"
+)
+
+func BenchmarkRequests(b *testing.B) {
+	for n := 2; n <= 32; n *= 2 {
+		b.Run("servers="+strconv.Itoa(n), func(b *testing.B) {
+			benchmarkGridRequests(b, n)
+		})
+	}
+}
+
+func benchmarkGridRequests(b *testing.B, n int) {
+	defer testlogger.T.SetErrorTB(b)()
+	errFatal := func(err error) {
+		b.Helper()
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+	rpc := NewSingleHandler[*testRequest, *testResponse](handlerTest2, newTestRequest, newTestResponse)
+	grid, err := SetupTestGrid(n)
+	errFatal(err)
+	b.Cleanup(grid.Cleanup)
+	// Create n managers.
+	for _, remote := range grid.Managers {
+		// Register a single handler which echos the payload.
+		errFatal(remote.RegisterSingleHandler(handlerTest, func(payload []byte) ([]byte, *RemoteErr) {
+			defer PutByteBuffer(payload)
+			return append(GetByteBuffer()[:0], payload...), nil
+		}))
+		errFatal(rpc.Register(remote, func(req *testRequest) (resp *testResponse, err *RemoteErr) {
+			return &testResponse{
+				OrgNum:    req.Num,
+				OrgString: req.String,
+				Embedded:  *req,
+			}, nil
+		}))
+		errFatal(err)
+	}
+	const payloadSize = 512
+	rng := rand.New(rand.NewSource(time.Now().UnixNano()))
+	payload := make([]byte, payloadSize)
+	_, err = rng.Read(payload)
+	errFatal(err)
+
+	// Wait for all to connect
+	// Parallel writes per server.
+	b.Run("bytes", func(b *testing.B) {
+		for par := 1; par <= 32; par *= 2 {
+			b.Run("par="+strconv.Itoa(par*runtime.GOMAXPROCS(0)), func(b *testing.B) {
+				defer timeout(60 * time.Second)()
+				ctx, cancel := context.WithTimeout(b.Context(), 30*time.Second)
+				defer cancel()
+				b.ReportAllocs()
+				b.SetBytes(int64(len(payload) * 2))
+				b.ResetTimer()
+				t := time.Now()
+				var ops int64
+				var lat int64
+				b.SetParallelism(par)
+				b.RunParallel(func(pb *testing.PB) {
+					rng := rand.New(rand.NewSource(time.Now().UnixNano()))
+					n := 0
+					var latency int64
+					managers := grid.Managers
+					hosts := grid.Hosts
+					for pb.Next() {
+						// Pick a random manager.
+						src, dst := rng.Intn(len(managers)), rng.Intn(len(managers))
+						if src == dst {
+							dst = (dst + 1) % len(managers)
+						}
+						local := managers[src]
+						conn := local.Connection(hosts[dst])
+						if conn == nil {
+							b.Fatal("No connection")
+						}
+						// Send the payload.
+						t := time.Now()
+						resp, err := conn.Request(ctx, handlerTest, payload)
+						latency += time.Since(t).Nanoseconds()
+						if err != nil {
+							if debugReqs {
+								fmt.Println(err.Error())
+							}
+							b.Fatal(err.Error())
+						}
+						PutByteBuffer(resp)
+						n++
+					}
+					atomic.AddInt64(&ops, int64(n))
+					atomic.AddInt64(&lat, latency)
+				})
+				spent := time.Since(t)
+				if spent > 0 && n > 0 {
+					// Since we are benchmarking n parallel servers we need to multiply by n.
+					// This will give an estimate of the total ops/s.
+					latency := float64(atomic.LoadInt64(&lat)) / float64(time.Millisecond)
+					b.ReportMetric(float64(n)*float64(ops)/spent.Seconds(), "vops/s")
+					b.ReportMetric(latency/float64(ops), "ms/op")
+				}
+			})
+		}
+	})
+	b.Run("rpc", func(b *testing.B) {
+		for par := 1; par <= 32; par *= 2 {
+			b.Run("par="+strconv.Itoa(par*runtime.GOMAXPROCS(0)), func(b *testing.B) {
+				defer timeout(60 * time.Second)()
+				ctx, cancel := context.WithTimeout(b.Context(), 30*time.Second)
+				defer cancel()
+				b.ReportAllocs()
+				b.ResetTimer()
+				t := time.Now()
+				var ops int64
+				var lat int64
+				b.SetParallelism(par)
+				b.RunParallel(func(pb *testing.PB) {
+					rng := rand.New(rand.NewSource(time.Now().UnixNano()))
+					n := 0
+					var latency int64
+					managers := grid.Managers
+					hosts := grid.Hosts
+					req := testRequest{
+						Num:    rng.Int(),
+						String: "hello",
+					}
+					for pb.Next() {
+						// Pick a random manager.
+						src, dst := rng.Intn(len(managers)), rng.Intn(len(managers))
+						if src == dst {
+							dst = (dst + 1) % len(managers)
+						}
+						local := managers[src]
+						conn := local.Connection(hosts[dst])
+						if conn == nil {
+							b.Fatal("No connection")
+						}
+						// Send the payload.
+						t := time.Now()
+						resp, err := rpc.Call(ctx, conn, &req)
+						latency += time.Since(t).Nanoseconds()
+						if err != nil {
+							if debugReqs {
+								fmt.Println(err.Error())
+							}
+							b.Fatal(err.Error())
+						}
+						rpc.PutResponse(resp)
+						n++
+					}
+					atomic.AddInt64(&ops, int64(n))
+					atomic.AddInt64(&lat, latency)
+				})
+				spent := time.Since(t)
+				if spent > 0 && n > 0 {
+					// Since we are benchmarking n parallel servers we need to multiply by n.
+					// This will give an estimate of the total ops/s.
+					latency := float64(atomic.LoadInt64(&lat)) / float64(time.Millisecond)
+					b.ReportMetric(float64(n)*float64(ops)/spent.Seconds(), "vops/s")
+					b.ReportMetric(latency/float64(ops), "ms/op")
+				}
+			})
+		}
+	})
+}
+
+func BenchmarkStream(b *testing.B) {
+	tests := []struct {
+		name string
+		fn   func(b *testing.B, n int)
+	}{
+		{name: "request", fn: benchmarkGridStreamReqOnly},
+		{name: "responses", fn: benchmarkGridStreamRespOnly},
+		{name: "twoway", fn: benchmarkGridStreamTwoway},
+	}
+	for _, test := range tests {
+		b.Run(test.name, func(b *testing.B) {
+			for n := 2; n <= 32; n *= 2 {
+				b.Run("servers="+strconv.Itoa(n), func(b *testing.B) {
+					test.fn(b, n)
+				})
+			}
+		})
+	}
+}
+
+func benchmarkGridStreamRespOnly(b *testing.B, n int) {
+	defer testlogger.T.SetErrorTB(b)()
+	errFatal := func(err error) {
+		b.Helper()
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+	grid, err := SetupTestGrid(n)
+	errFatal(err)
+	b.Cleanup(grid.Cleanup)
+	const responses = 10
+	// Create n managers.
+	for _, remote := range grid.Managers {
+		// Register a single handler which echos the payload.
+		errFatal(remote.RegisterStreamingHandler(handlerTest, StreamHandler{
+			// Send 10x response.
+			Handle: func(ctx context.Context, payload []byte, _ <-chan []byte, out chan<- []byte) *RemoteErr {
+				for i := range responses {
+					toSend := GetByteBuffer()[:0]
+					toSend = append(toSend, byte(i))
+					toSend = append(toSend, payload...)
+					select {
+					case <-ctx.Done():
+						return nil
+					case out <- toSend:
+					}
+				}
+				return nil
+			},
+
+			Subroute:    "some-subroute",
+			OutCapacity: 1, // Only one message buffered.
+			InCapacity:  0,
+		}))
+		errFatal(err)
+	}
+	const payloadSize = 512
+	rng := rand.New(rand.NewSource(time.Now().UnixNano()))
+	payload := make([]byte, payloadSize)
+	_, err = rng.Read(payload)
+	errFatal(err)
+
+	// Wait for all to connect
+	// Parallel writes per server.
+	for par := 1; par <= 32; par *= 2 {
+		b.Run("par="+strconv.Itoa(par*runtime.GOMAXPROCS(0)), func(b *testing.B) {
+			defer timeout(30 * time.Second)()
+			b.ReportAllocs()
+			b.SetBytes(int64(len(payload) * (responses + 1)))
+			b.ResetTimer()
+			t := time.Now()
+			var ops int64
+			var lat int64
+			b.SetParallelism(par)
+			b.RunParallel(func(pb *testing.PB) {
+				rng := rand.New(rand.NewSource(time.Now().UnixNano()))
+				n := 0
+				var latency int64
+				managers := grid.Managers
+				hosts := grid.Hosts
+				for pb.Next() {
+					// Pick a random manager.
+					src, dst := rng.Intn(len(managers)), rng.Intn(len(managers))
+					if src == dst {
+						dst = (dst + 1) % len(managers)
+					}
+					local := managers[src]
+					conn := local.Connection(hosts[dst]).Subroute("some-subroute")
+					if conn == nil {
+						b.Fatal("No connection")
+					}
+					ctx, cancel := context.WithTimeout(b.Context(), 30*time.Second)
+					// Send the payload.
+					t := time.Now()
+					st, err := conn.NewStream(ctx, handlerTest, payload)
+					if err != nil {
+						if debugReqs {
+							fmt.Println(err.Error())
+						}
+						b.Fatal(err.Error())
+					}
+					got := 0
+					err = st.Results(func(b []byte) error {
+						got++
+						PutByteBuffer(b)
+						return nil
+					})
+					if err != nil {
+						if debugReqs {
+							fmt.Println(err.Error())
+						}
+						b.Fatal(err.Error())
+					}
+					latency += time.Since(t).Nanoseconds()
+					cancel()
+					n += got
+				}
+				atomic.AddInt64(&ops, int64(n))
+				atomic.AddInt64(&lat, latency)
+			})
+			spent := time.Since(t)
+			if spent > 0 && n > 0 {
+				// Since we are benchmarking n parallel servers we need to multiply by n.
+				// This will give an estimate of the total ops/s.
+				latency := float64(atomic.LoadInt64(&lat)) / float64(time.Millisecond)
+				b.ReportMetric(float64(n)*float64(ops)/spent.Seconds(), "vops/s")
+				b.ReportMetric(latency/float64(ops), "ms/op")
+			}
+		})
+	}
+}
+
+func benchmarkGridStreamReqOnly(b *testing.B, n int) {
+	defer testlogger.T.SetErrorTB(b)()
+	errFatal := func(err error) {
+		b.Helper()
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+	grid, err := SetupTestGrid(n)
+	errFatal(err)
+	b.Cleanup(grid.Cleanup)
+	const requests = 10
+	// Create n managers.
+	for _, remote := range grid.Managers {
+		// Register a single handler which echos the payload.
+		errFatal(remote.RegisterStreamingHandler(handlerTest, StreamHandler{
+			// Send 10x requests.
+			Handle: func(ctx context.Context, payload []byte, in <-chan []byte, out chan<- []byte) *RemoteErr {
+				got := 0
+				for b := range in {
+					PutByteBuffer(b)
+					got++
+				}
+				if got != requests {
+					return NewRemoteErrf("wrong number of requests. want %d, got %d", requests, got)
+				}
+				return nil
+			},
+
+			Subroute:    "some-subroute",
+			OutCapacity: 1,
+			InCapacity:  1, // Only one message buffered.
+		}))
+		errFatal(err)
+	}
+	const payloadSize = 512
+	rng := rand.New(rand.NewSource(time.Now().UnixNano()))
+	payload := make([]byte, payloadSize)
+	_, err = rng.Read(payload)
+	errFatal(err)
+
+	// Wait for all to connect
+	// Parallel writes per server.
+	for par := 1; par <= 32; par *= 2 {
+		b.Run("par="+strconv.Itoa(par*runtime.GOMAXPROCS(0)), func(b *testing.B) {
+			defer timeout(30 * time.Second)()
+			b.ReportAllocs()
+			b.SetBytes(int64(len(payload) * (requests + 1)))
+			b.ResetTimer()
+			t := time.Now()
+			var ops int64
+			var lat int64
+			b.SetParallelism(par)
+			b.RunParallel(func(pb *testing.PB) {
+				rng := rand.New(rand.NewSource(time.Now().UnixNano()))
+				n := 0
+				var latency int64
+				managers := grid.Managers
+				hosts := grid.Hosts
+				for pb.Next() {
+					// Pick a random manager.
+					src, dst := rng.Intn(len(managers)), rng.Intn(len(managers))
+					if src == dst {
+						dst = (dst + 1) % len(managers)
+					}
+					local := managers[src]
+					conn := local.Connection(hosts[dst]).Subroute("some-subroute")
+					if conn == nil {
+						b.Fatal("No connection")
+					}
+					ctx, cancel := context.WithTimeout(b.Context(), 30*time.Second)
+					// Send the payload.
+					t := time.Now()
+					st, err := conn.NewStream(ctx, handlerTest, payload)
+					if err != nil {
+						if debugReqs {
+							fmt.Println(err.Error())
+						}
+						b.Fatal(err.Error())
+					}
+					got := 0
+					for range requests {
+						got++
+						st.Requests <- append(GetByteBuffer()[:0], payload...)
+					}
+					close(st.Requests)
+					err = st.Results(func(b []byte) error {
+						return nil
+					})
+					if err != nil {
+						if debugReqs {
+							fmt.Println(err.Error())
+						}
+						b.Fatal(err.Error())
+					}
+					latency += time.Since(t).Nanoseconds()
+					cancel()
+					n += got
+				}
+				atomic.AddInt64(&ops, int64(n))
+				atomic.AddInt64(&lat, latency)
+			})
+			spent := time.Since(t)
+			if spent > 0 && n > 0 {
+				// Since we are benchmarking n parallel servers we need to multiply by n.
+				// This will give an estimate of the total ops/s.
+				latency := float64(atomic.LoadInt64(&lat)) / float64(time.Millisecond)
+				b.ReportMetric(float64(n)*float64(ops)/spent.Seconds(), "vops/s")
+				b.ReportMetric(latency/float64(ops), "ms/op")
+			}
+		})
+	}
+}
+
+func benchmarkGridStreamTwoway(b *testing.B, n int) {
+	defer testlogger.T.SetErrorTB(b)()
+
+	errFatal := func(err error) {
+		b.Helper()
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+	grid, err := SetupTestGrid(n)
+	errFatal(err)
+	b.Cleanup(grid.Cleanup)
+	const messages = 10
+	// Create n managers.
+	const payloadSize = 512
+	rng := rand.New(rand.NewSource(time.Now().UnixNano()))
+	payload := make([]byte, payloadSize)
+	_, err = rng.Read(payload)
+	errFatal(err)
+
+	for _, remote := range grid.Managers {
+		// Register a single handler which echos the payload.
+		errFatal(remote.RegisterStreamingHandler(handlerTest, StreamHandler{
+			// Send 10x requests.
+			Handle: func(ctx context.Context, payload []byte, in <-chan []byte, out chan<- []byte) *RemoteErr {
+				got := 0
+				for b := range in {
+					out <- b
+					got++
+				}
+				if got != messages {
+					return NewRemoteErrf("wrong number of requests. want %d, got %d", messages, got)
+				}
+				return nil
+			},
+
+			Subroute:    "some-subroute",
+			OutCapacity: 1,
+			InCapacity:  1, // Only one message buffered.
+		}))
+		errFatal(err)
+	}
+
+	// Wait for all to connect
+	// Parallel writes per server.
+	for par := 1; par <= 32; par *= 2 {
+		b.Run("par="+strconv.Itoa(par*runtime.GOMAXPROCS(0)), func(b *testing.B) {
+			defer timeout(30 * time.Second)()
+			b.ReportAllocs()
+			b.SetBytes(int64(len(payload) * (2*messages + 1)))
+			b.ResetTimer()
+			t := time.Now()
+			var ops int64
+			var lat int64
+			b.SetParallelism(par)
+			b.RunParallel(func(pb *testing.PB) {
+				rng := rand.New(rand.NewSource(time.Now().UnixNano()))
+				n := 0
+				var latency int64
+				managers := grid.Managers
+				hosts := grid.Hosts
+				for pb.Next() {
+					// Pick a random manager.
+					src, dst := rng.Intn(len(managers)), rng.Intn(len(managers))
+					if src == dst {
+						dst = (dst + 1) % len(managers)
+					}
+					local := managers[src]
+					conn := local.Connection(hosts[dst]).Subroute("some-subroute")
+					if conn == nil {
+						b.Fatal("No connection")
+					}
+					ctx, cancel := context.WithTimeout(b.Context(), 30*time.Second)
+					// Send the payload.
+					t := time.Now()
+					st, err := conn.NewStream(ctx, handlerTest, payload)
+					if err != nil {
+						if debugReqs {
+							fmt.Println(err.Error())
+						}
+						b.Fatal(err.Error())
+					}
+					got := 0
+					sent := 0
+					go func() {
+						for range messages {
+							st.Requests <- append(GetByteBuffer()[:0], payload...)
+							if sent++; sent == messages {
+								close(st.Requests)
+								return
+							}
+						}
+					}()
+					err = st.Results(func(b []byte) error {
+						got++
+						PutByteBuffer(b)
+						return nil
+					})
+					if err != nil {
+						if debugReqs {
+							fmt.Println(err.Error())
+						}
+						b.Fatal(err.Error())
+					}
+					if got != messages {
+						b.Fatalf("wrong number of responses. want %d, got %d", messages, got)
+					}
+					latency += time.Since(t).Nanoseconds()
+					cancel()
+					n += got
+				}
+				atomic.AddInt64(&ops, int64(n*2))
+				atomic.AddInt64(&lat, latency)
+			})
+			spent := time.Since(t)
+			if spent > 0 && n > 0 {
+				// Since we are benchmarking n parallel servers we need to multiply by n.
+				// This will give an estimate of the total ops/s.
+				latency := float64(atomic.LoadInt64(&lat)) / float64(time.Millisecond)
+				b.ReportMetric(float64(n)*float64(ops)/spent.Seconds(), "vops/s")
+				b.ReportMetric(latency/float64(ops), "ms/op")
+			}
+		})
+	}
+}

internal/grid/connection_test.go

@@ -0,0 +1,209 @@
+// Copyright (c) 2015-2023 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package grid
+
+import (
+	"context"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"github.com/minio/minio/internal/logger/target/testlogger"
+)
+
+func TestDisconnect(t *testing.T) {
+	defer testlogger.T.SetLogTB(t)()
+	defer timeout(10 * time.Second)()
+	hosts, listeners, _ := getHosts(2)
+	dialer := &net.Dialer{
+		Timeout: 1 * time.Second,
+	}
+	errFatal := func(err error) {
+		t.Helper()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+	wrapServer := func(handler http.Handler) http.Handler {
+		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			t.Logf("Got a %s request for: %v", r.Method, r.URL)
+			handler.ServeHTTP(w, r)
+		})
+	}
+	connReady := make(chan struct{})
+	// We fake a local and remote server.
+	localHost := hosts[0]
+	remoteHost := hosts[1]
+	local, err := NewManager(t.Context(), ManagerOptions{
+		Dialer: ConnectWS(dialer.DialContext,
+			dummyNewToken,
+			nil),
+		Local:        localHost,
+		Hosts:        hosts,
+		AuthFn:       dummyNewToken,
+		AuthToken:    dummyTokenValidate,
+		BlockConnect: connReady,
+	})
+	errFatal(err)
+
+	// 1: Echo
+	errFatal(local.RegisterSingleHandler(handlerTest, func(payload []byte) ([]byte, *RemoteErr) {
+		t.Log("1: server payload: ", len(payload), "bytes.")
+		return append([]byte{}, payload...), nil
+	}))
+	// 2: Return as error
+	errFatal(local.RegisterSingleHandler(handlerTest2, func(payload []byte) ([]byte, *RemoteErr) {
+		t.Log("2: server payload: ", len(payload), "bytes.")
+		err := RemoteErr(payload)
+		return nil, &err
+	}))
+
+	remote, err := NewManager(t.Context(), ManagerOptions{
+		Dialer: ConnectWS(dialer.DialContext,
+			dummyNewToken,
+			nil),
+		Local:        remoteHost,
+		Hosts:        hosts,
+		AuthFn:       dummyNewToken,
+		AuthToken:    dummyTokenValidate,
+		BlockConnect: connReady,
+	})
+	errFatal(err)
+
+	localServer := startServer(t, listeners[0], wrapServer(local.Handler(dummyRequestValidate)))
+	remoteServer := startServer(t, listeners[1], wrapServer(remote.Handler(dummyRequestValidate)))
+	close(connReady)
+
+	defer func() {
+		local.debugMsg(debugShutdown)
+		remote.debugMsg(debugShutdown)
+		remoteServer.Close()
+		localServer.Close()
+		remote.debugMsg(debugWaitForExit)
+		local.debugMsg(debugWaitForExit)
+	}()
+
+	cleanReqs := make(chan struct{})
+	gotCall := make(chan struct{})
+	defer close(cleanReqs)
+	// 1: Block forever
+	h1 := func(payload []byte) ([]byte, *RemoteErr) {
+		gotCall <- struct{}{}
+		<-cleanReqs
+		return nil, nil
+	}
+	// 2: Also block, but with streaming.
+	h2 := StreamHandler{
+		Handle: func(ctx context.Context, payload []byte, request <-chan []byte, resp chan<- []byte) *RemoteErr {
+			gotCall <- struct{}{}
+			select {
+			case <-ctx.Done():
+				gotCall <- struct{}{}
+			case <-cleanReqs:
+				panic("should not be called")
+			}
+			return nil
+		},
+		OutCapacity: 1,
+		InCapacity:  1,
+	}
+	errFatal(remote.RegisterSingleHandler(handlerTest, h1))
+	errFatal(remote.RegisterStreamingHandler(handlerTest2, h2))
+	errFatal(local.RegisterSingleHandler(handlerTest, h1))
+	errFatal(local.RegisterStreamingHandler(handlerTest2, h2))
+
+	// local to remote
+	remoteConn := local.Connection(remoteHost)
+	errFatal(remoteConn.WaitForConnect(t.Context()))
+	const testPayload = "Hello Grid World!"
+
+	gotResp := make(chan struct{})
+	go func() {
+		start := time.Now()
+		t.Log("Roundtrip: sending request")
+		resp, err := remoteConn.Request(t.Context(), handlerTest, []byte(testPayload))
+		t.Log("Roundtrip:", time.Since(start), resp, err)
+		gotResp <- struct{}{}
+	}()
+	<-gotCall
+	remote.debugMsg(debugKillInbound)
+	local.debugMsg(debugKillInbound)
+	<-gotResp
+
+	// Must reconnect
+	errFatal(remoteConn.WaitForConnect(t.Context()))
+
+	stream, err := remoteConn.NewStream(t.Context(), handlerTest2, []byte(testPayload))
+	errFatal(err)
+	go func() {
+		for resp := range stream.responses {
+			t.Log("Resp:", resp, err)
+		}
+		gotResp <- struct{}{}
+	}()
+
+	<-gotCall
+	remote.debugMsg(debugKillOutbound)
+	local.debugMsg(debugKillOutbound)
+	errFatal(remoteConn.WaitForConnect(t.Context()))
+
+	<-gotResp
+	// Killing should cancel the context on the request.
+	<-gotCall
+}
+
+func TestShouldConnect(t *testing.T) {
+	var c Connection
+	var cReverse Connection
+	hosts := []string{"a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "x", "y", "z", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9"}
+	for x := range hosts {
+		should := 0
+		for y := range hosts {
+			if x == y {
+				continue
+			}
+			c.Local = hosts[x]
+			c.Remote = hosts[y]
+			cReverse.Local = hosts[y]
+			cReverse.Remote = hosts[x]
+			if c.shouldConnect() == cReverse.shouldConnect() {
+				t.Errorf("shouldConnect(%q, %q) != shouldConnect(%q, %q)", hosts[x], hosts[y], hosts[y], hosts[x])
+			}
+			if c.shouldConnect() {
+				should++
+			}
+		}
+		if should < 10 {
+			t.Errorf("host %q only connects to %d hosts", hosts[x], should)
+		}
+		t.Logf("host %q should connect to %d hosts", hosts[x], should)
+	}
+}
+
+func startServer(t testing.TB, listener net.Listener, handler http.Handler) (server *httptest.Server) {
+	t.Helper()
+	server = httptest.NewUnstartedServer(handler)
+	server.Config.Addr = listener.Addr().String()
+	server.Listener = listener
+	server.Start()
+	// t.Cleanup(server.Close)
+	t.Log("Started server on", server.Config.Addr, "URL:", server.URL)
+	return server
+}

internal/grid/debug.go

@@ -0,0 +1,181 @@
+// Copyright (c) 2015-2023 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package grid
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"sync"
+	"time"
+
+	"github.com/minio/mux"
+)
+
+//go:generate stringer -type=debugMsg $GOFILE
+
+// debugMsg is a debug message for testing purposes.
+// may only be used for tests.
+type debugMsg int
+
+const (
+	debugPrint = false
+	debugReqs  = false
+)
+
+const (
+	debugShutdown debugMsg = iota
+	debugKillInbound
+	debugKillOutbound
+	debugWaitForExit
+	debugSetConnPingDuration
+	debugSetClientPingDuration
+	debugAddToDeadline
+	debugIsOutgoingClosed
+	debugBlockInboundMessages
+)
+
+// TestGrid contains a grid of servers for testing purposes.
+type TestGrid struct {
+	Servers     []*httptest.Server
+	Listeners   []net.Listener
+	Managers    []*Manager
+	Mux         []*mux.Router
+	Hosts       []string
+	cleanupOnce sync.Once
+	cancel      context.CancelFunc
+}
+
+// SetupTestGrid creates a new grid for testing purposes.
+// Select the number of hosts to create.
+// Call (TestGrid).Cleanup() when done.
+func SetupTestGrid(n int) (*TestGrid, error) {
+	hosts, listeners, err := getHosts(n)
+	if err != nil {
+		return nil, err
+	}
+	dialer := &net.Dialer{
+		Timeout: 5 * time.Second,
+	}
+	var res TestGrid
+	res.Hosts = hosts
+	ready := make(chan struct{})
+	ctx, cancel := context.WithCancel(context.Background())
+	res.cancel = cancel
+	for i, host := range hosts {
+		manager, err := NewManager(ctx, ManagerOptions{
+			Dialer: ConnectWS(dialer.DialContext,
+				dummyNewToken,
+				nil),
+			Local:        host,
+			Hosts:        hosts,
+			AuthFn:       dummyNewToken,
+			AuthToken:    dummyTokenValidate,
+			BlockConnect: ready,
+			RoutePath:    RoutePath,
+		})
+		if err != nil {
+			return nil, err
+		}
+		m := mux.NewRouter()
+		m.Handle(RoutePath, manager.Handler(dummyRequestValidate))
+		res.Managers = append(res.Managers, manager)
+		res.Servers = append(res.Servers, startHTTPServer(listeners[i], m))
+		res.Listeners = append(res.Listeners, listeners[i])
+		res.Mux = append(res.Mux, m)
+	}
+	close(ready)
+	for _, m := range res.Managers {
+		for _, remote := range m.Targets() {
+			if err := m.Connection(remote).WaitForConnect(ctx); err != nil {
+				return nil, err
+			}
+		}
+	}
+	return &res, nil
+}
+
+// Cleanup will clean up the test grid.
+func (t *TestGrid) Cleanup() {
+	t.cancel()
+	t.cleanupOnce.Do(func() {
+		for _, manager := range t.Managers {
+			manager.debugMsg(debugShutdown)
+		}
+		for _, server := range t.Servers {
+			server.Close()
+		}
+		for _, listener := range t.Listeners {
+			listener.Close()
+		}
+	})
+}
+
+// WaitAllConnect will wait for all connections to be established.
+func (t *TestGrid) WaitAllConnect(ctx context.Context) {
+	for _, manager := range t.Managers {
+		for _, remote := range manager.Targets() {
+			if manager.HostName() == remote {
+				continue
+			}
+			if err := manager.Connection(remote).WaitForConnect(ctx); err != nil {
+				panic(err)
+			}
+		}
+	}
+}
+
+func getHosts(n int) (hosts []string, listeners []net.Listener, err error) {
+	for range n {
+		l, err := net.Listen("tcp", "127.0.0.1:0")
+		if err != nil {
+			if l, err = net.Listen("tcp6", "[::1]:0"); err != nil {
+				return nil, nil, fmt.Errorf("httptest: failed to listen on a port: %v", err)
+			}
+		}
+		addr := l.Addr()
+		hosts = append(hosts, "http://"+addr.String())
+		listeners = append(listeners, l)
+	}
+	return hosts, listeners, err
+}
+
+func startHTTPServer(listener net.Listener, handler http.Handler) (server *httptest.Server) {
+	server = httptest.NewUnstartedServer(handler)
+	server.Config.Addr = listener.Addr().String()
+	server.Listener = listener
+	server.Start()
+	return server
+}
+
+func dummyRequestValidate(r *http.Request) error {
+	return nil
+}
+
+func dummyTokenValidate(token string) error {
+	if token == "debug" {
+		return nil
+	}
+	return fmt.Errorf("invalid token. want empty, got %s", token)
+}
+
+func dummyNewToken() string {
+	return "debug"
+}

internal/grid/debugmsg_string.go

@@ -0,0 +1,31 @@
+// Code generated by "stringer -type=debugMsg debug.go"; DO NOT EDIT.
+
+package grid
+
+import "strconv"
+
+func _() {
+	// An "invalid array index" compiler error signifies that the constant values have changed.
+	// Re-run the stringer command to generate them again.
+	var x [1]struct{}
+	_ = x[debugShutdown-0]
+	_ = x[debugKillInbound-1]
+	_ = x[debugKillOutbound-2]
+	_ = x[debugWaitForExit-3]
+	_ = x[debugSetConnPingDuration-4]
+	_ = x[debugSetClientPingDuration-5]
+	_ = x[debugAddToDeadline-6]
+	_ = x[debugIsOutgoingClosed-7]
+	_ = x[debugBlockInboundMessages-8]
+}
+
+const _debugMsg_name = "debugShutdowndebugKillInbounddebugKillOutbounddebugWaitForExitdebugSetConnPingDurationdebugSetClientPingDurationdebugAddToDeadlinedebugIsOutgoingCloseddebugBlockInboundMessages"
+
+var _debugMsg_index = [...]uint8{0, 13, 29, 46, 62, 86, 112, 130, 151, 176}
+
+func (i debugMsg) String() string {
+	if i < 0 || i >= debugMsg(len(_debugMsg_index)-1) {
+		return "debugMsg(" + strconv.FormatInt(int64(i), 10) + ")"
+	}
+	return _debugMsg_name[_debugMsg_index[i]:_debugMsg_index[i+1]]
+}

internal/grid/errors.go

@@ -0,0 +1,43 @@
+// Copyright (c) 2015-2023 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package grid
+
+import (
+	"errors"
+	"fmt"
+)
+
+var (
+	// ErrUnknownHandler is returned when an unknown handler is requested.
+	ErrUnknownHandler = errors.New("unknown mux handler")
+
+	// ErrHandlerAlreadyExists is returned when a handler is already registered.
+	ErrHandlerAlreadyExists = errors.New("mux handler already exists")
+
+	// ErrIncorrectSequence is returned when an out-of-sequence item is received.
+	ErrIncorrectSequence = errors.New("out-of-sequence item received")
+)
+
+// ErrResponse is a remote error response.
+type ErrResponse struct {
+	msg string
+}
+
+func (e ErrResponse) Error() string {
+	return fmt.Sprintf("remote: %s", e.msg)
+}

internal/grid/grid.go

@@ -0,0 +1,243 @@
+// Copyright (c) 2015-2023 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+// Package grid provides single-connection two-way grid communication.
+package grid
+
+import (
+	"context"
+	"crypto/tls"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/gobwas/ws"
+	"github.com/gobwas/ws/wsutil"
+	"github.com/minio/minio/internal/bpool"
+)
+
+// ErrDisconnected is returned when the connection to the remote has been lost during the call.
+var ErrDisconnected = RemoteErr("remote disconnected")
+
+const (
+	// minBufferSize is the minimum buffer size.
+	// Buffers below this is not reused.
+	minBufferSize = 1 << 10
+
+	// defaultBufferSize is the default buffer allocation size.
+	defaultBufferSize = 4 << 10
+
+	// maxBufferSize is the maximum buffer size.
+	// Buffers larger than this is not reused.
+	maxBufferSize = 96 << 10
+
+	// This is the assumed size of bigger buffers and allocation size.
+	biggerBufMin = 32 << 10
+
+	// This is the maximum size of bigger buffers.
+	biggerBufMax = maxBufferSize
+
+	// If there is a queue, merge up to this many messages.
+	maxMergeMessages = 50
+
+	// clientPingInterval will ping the remote handler every 15 seconds.
+	// Clients disconnect when we exceed 2 intervals.
+	clientPingInterval = 15 * time.Second
+
+	// Deadline for single (non-streaming) requests to complete.
+	// Used if no deadline is provided on context.
+	defaultSingleRequestTimeout = time.Minute
+)
+
+var internalByteBuffer = bpool.Pool[*[]byte]{
+	New: func() *[]byte {
+		m := make([]byte, 0, defaultBufferSize)
+		return &m
+	},
+}
+
+var internal32KByteBuffer = bpool.Pool[*[]byte]{
+	New: func() *[]byte {
+		m := make([]byte, 0, biggerBufMin)
+		return &m
+	},
+}
+
+// GetByteBuffer can be replaced with a function that returns a small
+// byte buffer.
+// When replacing PutByteBuffer should also be replaced
+// There is no minimum size.
+var GetByteBuffer = func() []byte {
+	b := *internalByteBuffer.Get()
+	return b[:0]
+}
+
+// GetByteBufferCap returns a length 0 byte buffer with at least the given capacity.
+func GetByteBufferCap(wantSz int) []byte {
+	if wantSz < defaultBufferSize {
+		b := GetByteBuffer()[:0]
+		if cap(b) >= wantSz {
+			return b
+		}
+		PutByteBuffer(b)
+	}
+	if wantSz <= maxBufferSize {
+		b := *internal32KByteBuffer.Get()
+		if cap(b) >= wantSz {
+			return b[:0]
+		}
+		internal32KByteBuffer.Put(&b)
+	}
+	return make([]byte, 0, wantSz)
+}
+
+// PutByteBuffer is for returning byte buffers.
+var PutByteBuffer = func(b []byte) {
+	if cap(b) >= biggerBufMin && cap(b) < biggerBufMax {
+		internal32KByteBuffer.Put(&b)
+		return
+	}
+	if cap(b) >= minBufferSize && cap(b) < biggerBufMin {
+		internalByteBuffer.Put(&b)
+		return
+	}
+}
+
+// readAllInto reads from r and appends to b until an error or EOF and returns the data it read.
+// A successful call returns err == nil, not err == EOF. Because readAllInto is
+// defined to read from src until EOF, it does not treat an EOF from Read
+// as an error to be reported.
+func readAllInto(b []byte, r *wsutil.Reader, want int64) ([]byte, error) {
+	read := int64(0)
+	for {
+		if len(b) == cap(b) {
+			// Add more capacity (let append pick how much).
+			b = append(b, 0)[:len(b)]
+		}
+		n, err := r.Read(b[len(b):cap(b)])
+		b = b[:len(b)+n]
+		if err != nil {
+			if errors.Is(err, io.EOF) {
+				if want >= 0 && read+int64(n) != want {
+					return nil, io.ErrUnexpectedEOF
+				}
+				err = nil
+			}
+			return b, err
+		}
+		read += int64(n)
+		if want >= 0 && read == want {
+			// No need to read more...
+			return b, nil
+		}
+	}
+}
+
+// getDeadline will truncate the deadline so it is at least 1ms and at most MaxDeadline.
+func getDeadline(d time.Duration) time.Duration {
+	if d < time.Millisecond {
+		return 0
+	}
+	if d > MaxDeadline {
+		return MaxDeadline
+	}
+	return d
+}
+
+type writerWrapper struct {
+	ch  chan<- []byte
+	ctx context.Context
+}
+
+func (w *writerWrapper) Write(p []byte) (n int, err error) {
+	buf := GetByteBufferCap(len(p))
+	buf = buf[:len(p)]
+	copy(buf, p)
+	select {
+	case w.ch <- buf:
+		return len(p), nil
+	case <-w.ctx.Done():
+		return 0, context.Cause(w.ctx)
+	}
+}
+
+// WriterToChannel will return an io.Writer that writes to the given channel.
+// The context both allows returning errors on writes and to ensure that
+// this isn't abandoned if the channel is no longer being read from.
+func WriterToChannel(ctx context.Context, ch chan<- []byte) io.Writer {
+	return &writerWrapper{ch: ch, ctx: ctx}
+}
+
+// bytesOrLength returns small (<=100b) byte slices as string, otherwise length.
+func bytesOrLength(b []byte) string {
+	if len(b) > 100 {
+		return fmt.Sprintf("%d bytes", len(b))
+	}
+	return fmt.Sprint(string(b))
+}
+
+// ConnDialer is a function that dials a connection to the given address.
+// There should be no retries in this function,
+// and should have a timeout of something like 2 seconds.
+// The returned net.Conn should also have quick disconnect on errors.
+// The net.Conn must support all features as described by the net.Conn interface.
+type ConnDialer func(ctx context.Context, address string) (net.Conn, error)
+
+// ConnectWSWithRoutePath is like ConnectWS but with a custom grid route path.
+func ConnectWSWithRoutePath(dial ContextDialer, auth AuthFn, tls *tls.Config, routePath string) func(ctx context.Context, remote string) (net.Conn, error) {
+	return func(ctx context.Context, remote string) (net.Conn, error) {
+		toDial := strings.Replace(remote, "http://", "ws://", 1)
+		toDial = strings.Replace(toDial, "https://", "wss://", 1)
+		toDial += routePath
+
+		dialer := ws.DefaultDialer
+		dialer.ReadBufferSize = readBufferSize
+		dialer.WriteBufferSize = writeBufferSize
+		dialer.Timeout = defaultDialTimeout
+		if dial != nil {
+			dialer.NetDial = dial
+		}
+		header := make(http.Header, 2)
+		header.Set("Authorization", "Bearer "+auth())
+		header.Set("X-Minio-Time", strconv.FormatInt(time.Now().UnixNano(), 10))
+
+		if len(header) > 0 {
+			dialer.Header = ws.HandshakeHeaderHTTP(header)
+		}
+		dialer.TLSConfig = tls
+
+		conn, br, _, err := dialer.Dial(ctx, toDial)
+		if br != nil {
+			ws.PutReader(br)
+		}
+		return conn, err
+	}
+}
+
+// ConnectWS returns a function that dials a websocket connection to the given address.
+// Route and auth are added to the connection.
+func ConnectWS(dial ContextDialer, auth AuthFn, tls *tls.Config) func(ctx context.Context, remote string) (net.Conn, error) {
+	return ConnectWSWithRoutePath(dial, auth, tls, RoutePath)
+}
+
+// ValidateTokenFn must validate the token and return an error if it is invalid.
+type ValidateTokenFn func(token string) error

internal/grid/grid_types_msgp_test.go

@@ -0,0 +1,368 @@
+// Code generated by github.com/tinylib/msgp DO NOT EDIT.
+
+package grid
+
+import (
+	"github.com/tinylib/msgp/msgp"
+)
+
+// DecodeMsg implements msgp.Decodable
+func (z *testRequest) DecodeMsg(dc *msgp.Reader) (err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, err = dc.ReadMapHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, err = dc.ReadMapKeyPtr()
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "Num":
+			z.Num, err = dc.ReadInt()
+			if err != nil {
+				err = msgp.WrapError(err, "Num")
+				return
+			}
+		case "String":
+			z.String, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "String")
+				return
+			}
+		default:
+			err = dc.Skip()
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z testRequest) EncodeMsg(en *msgp.Writer) (err error) {
+	// map header, size 2
+	// write "Num"
+	err = en.Append(0x82, 0xa3, 0x4e, 0x75, 0x6d)
+	if err != nil {
+		return
+	}
+	err = en.WriteInt(z.Num)
+	if err != nil {
+		err = msgp.WrapError(err, "Num")
+		return
+	}
+	// write "String"
+	err = en.Append(0xa6, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.String)
+	if err != nil {
+		err = msgp.WrapError(err, "String")
+		return
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z testRequest) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// map header, size 2
+	// string "Num"
+	o = append(o, 0x82, 0xa3, 0x4e, 0x75, 0x6d)
+	o = msgp.AppendInt(o, z.Num)
+	// string "String"
+	o = append(o, 0xa6, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67)
+	o = msgp.AppendString(o, z.String)
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *testRequest) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "Num":
+			z.Num, bts, err = msgp.ReadIntBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Num")
+				return
+			}
+		case "String":
+			z.String, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "String")
+				return
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z testRequest) Msgsize() (s int) {
+	s = 1 + 4 + msgp.IntSize + 7 + msgp.StringPrefixSize + len(z.String)
+	return
+}
+
+// DecodeMsg implements msgp.Decodable
+func (z *testResponse) DecodeMsg(dc *msgp.Reader) (err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, err = dc.ReadMapHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, err = dc.ReadMapKeyPtr()
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "OrgNum":
+			z.OrgNum, err = dc.ReadInt()
+			if err != nil {
+				err = msgp.WrapError(err, "OrgNum")
+				return
+			}
+		case "OrgString":
+			z.OrgString, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "OrgString")
+				return
+			}
+		case "Embedded":
+			var zb0002 uint32
+			zb0002, err = dc.ReadMapHeader()
+			if err != nil {
+				err = msgp.WrapError(err, "Embedded")
+				return
+			}
+			for zb0002 > 0 {
+				zb0002--
+				field, err = dc.ReadMapKeyPtr()
+				if err != nil {
+					err = msgp.WrapError(err, "Embedded")
+					return
+				}
+				switch msgp.UnsafeString(field) {
+				case "Num":
+					z.Embedded.Num, err = dc.ReadInt()
+					if err != nil {
+						err = msgp.WrapError(err, "Embedded", "Num")
+						return
+					}
+				case "String":
+					z.Embedded.String, err = dc.ReadString()
+					if err != nil {
+						err = msgp.WrapError(err, "Embedded", "String")
+						return
+					}
+				default:
+					err = dc.Skip()
+					if err != nil {
+						err = msgp.WrapError(err, "Embedded")
+						return
+					}
+				}
+			}
+		default:
+			err = dc.Skip()
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z *testResponse) EncodeMsg(en *msgp.Writer) (err error) {
+	// map header, size 3
+	// write "OrgNum"
+	err = en.Append(0x83, 0xa6, 0x4f, 0x72, 0x67, 0x4e, 0x75, 0x6d)
+	if err != nil {
+		return
+	}
+	err = en.WriteInt(z.OrgNum)
+	if err != nil {
+		err = msgp.WrapError(err, "OrgNum")
+		return
+	}
+	// write "OrgString"
+	err = en.Append(0xa9, 0x4f, 0x72, 0x67, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.OrgString)
+	if err != nil {
+		err = msgp.WrapError(err, "OrgString")
+		return
+	}
+	// write "Embedded"
+	err = en.Append(0xa8, 0x45, 0x6d, 0x62, 0x65, 0x64, 0x64, 0x65, 0x64)
+	if err != nil {
+		return
+	}
+	// map header, size 2
+	// write "Num"
+	err = en.Append(0x82, 0xa3, 0x4e, 0x75, 0x6d)
+	if err != nil {
+		return
+	}
+	err = en.WriteInt(z.Embedded.Num)
+	if err != nil {
+		err = msgp.WrapError(err, "Embedded", "Num")
+		return
+	}
+	// write "String"
+	err = en.Append(0xa6, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.Embedded.String)
+	if err != nil {
+		err = msgp.WrapError(err, "Embedded", "String")
+		return
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z *testResponse) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// map header, size 3
+	// string "OrgNum"
+	o = append(o, 0x83, 0xa6, 0x4f, 0x72, 0x67, 0x4e, 0x75, 0x6d)
+	o = msgp.AppendInt(o, z.OrgNum)
+	// string "OrgString"
+	o = append(o, 0xa9, 0x4f, 0x72, 0x67, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67)
+	o = msgp.AppendString(o, z.OrgString)
+	// string "Embedded"
+	o = append(o, 0xa8, 0x45, 0x6d, 0x62, 0x65, 0x64, 0x64, 0x65, 0x64)
+	// map header, size 2
+	// string "Num"
+	o = append(o, 0x82, 0xa3, 0x4e, 0x75, 0x6d)
+	o = msgp.AppendInt(o, z.Embedded.Num)
+	// string "String"
+	o = append(o, 0xa6, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67)
+	o = msgp.AppendString(o, z.Embedded.String)
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *testResponse) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "OrgNum":
+			z.OrgNum, bts, err = msgp.ReadIntBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "OrgNum")
+				return
+			}
+		case "OrgString":
+			z.OrgString, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "OrgString")
+				return
+			}
+		case "Embedded":
+			var zb0002 uint32
+			zb0002, bts, err = msgp.ReadMapHeaderBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Embedded")
+				return
+			}
+			for zb0002 > 0 {
+				zb0002--
+				field, bts, err = msgp.ReadMapKeyZC(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "Embedded")
+					return
+				}
+				switch msgp.UnsafeString(field) {
+				case "Num":
+					z.Embedded.Num, bts, err = msgp.ReadIntBytes(bts)
+					if err != nil {
+						err = msgp.WrapError(err, "Embedded", "Num")
+						return
+					}
+				case "String":
+					z.Embedded.String, bts, err = msgp.ReadStringBytes(bts)
+					if err != nil {
+						err = msgp.WrapError(err, "Embedded", "String")
+						return
+					}
+				default:
+					bts, err = msgp.Skip(bts)
+					if err != nil {
+						err = msgp.WrapError(err, "Embedded")
+						return
+					}
+				}
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *testResponse) Msgsize() (s int) {
+	s = 1 + 7 + msgp.IntSize + 10 + msgp.StringPrefixSize + len(z.OrgString) + 9 + 1 + 4 + msgp.IntSize + 7 + msgp.StringPrefixSize + len(z.Embedded.String)
+	return
+}

internal/grid/grid_types_test.go

@@ -0,0 +1,39 @@
+// Copyright (c) 2015-2023 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package grid
+
+//go:generate msgp -unexported -file=$GOFILE -tests=false -o=grid_types_msgp_test.go
+
+type testRequest struct {
+	Num    int
+	String string
+}
+
+type testResponse struct {
+	OrgNum    int
+	OrgString string
+	Embedded  testRequest
+}
+
+func newTestRequest() *testRequest {
+	return &testRequest{}
+}
+
+func newTestResponse() *testResponse {
+	return &testResponse{}
+}

internal/grid/handlers.go

@@ -0,0 +1,907 @@
+// Copyright (c) 2015-2023 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package grid
+
+import (
+	"context"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"strings"
+
+	"github.com/minio/minio/internal/bpool"
+	"github.com/minio/minio/internal/hash/sha256"
+	xioutil "github.com/minio/minio/internal/ioutil"
+	"github.com/tinylib/msgp/msgp"
+)
+
+//go:generate stringer -type=HandlerID -output=handlers_string.go -trimprefix=Handler msg.go $GOFILE
+
+// HandlerID is a handler identifier.
+// It is used to determine request routing on the server.
+// Handlers can be registered with a static subroute.
+// Do NOT remove or change the order of existing handlers.
+const (
+	// handlerInvalid is reserved to check for uninitialized values.
+	handlerInvalid HandlerID = iota
+	HandlerLockLock
+	HandlerLockRLock
+	HandlerLockUnlock
+	HandlerLockRUnlock
+	HandlerLockRefresh
+	HandlerLockForceUnlock
+	HandlerWalkDir
+	HandlerStatVol
+	HandlerDiskInfo
+	HandlerNSScanner
+	HandlerReadXL
+	HandlerReadVersion
+	HandlerDeleteFile
+	HandlerDeleteVersion
+	HandlerUpdateMetadata
+	HandlerWriteMetadata
+	HandlerCheckParts
+	HandlerRenameData
+	HandlerRenameFile
+	HandlerReadAll
+	HandlerServerVerify
+	HandlerTrace
+	HandlerListen
+	HandlerDeleteBucketMetadata
+	HandlerLoadBucketMetadata
+	HandlerReloadSiteReplicationConfig
+	HandlerReloadPoolMeta
+	HandlerStopRebalance
+	HandlerLoadRebalanceMeta
+	HandlerLoadTransitionTierConfig
+	HandlerDeletePolicy
+	HandlerLoadPolicy
+	HandlerLoadPolicyMapping
+	HandlerDeleteServiceAccount
+	HandlerLoadServiceAccount
+	HandlerDeleteUser
+	HandlerLoadUser
+	HandlerLoadGroup
+	HandlerHealBucket
+	HandlerMakeBucket
+	HandlerHeadBucket
+	HandlerDeleteBucket
+	HandlerGetMetrics
+	HandlerGetResourceMetrics
+	HandlerGetMemInfo
+	HandlerGetProcInfo
+	HandlerGetOSInfo
+	HandlerGetPartitions
+	HandlerGetNetInfo
+	HandlerGetCPUs
+	HandlerServerInfo
+	HandlerGetSysConfig
+	HandlerGetSysServices
+	HandlerGetSysErrors
+	HandlerGetAllBucketStats
+	HandlerGetBucketStats
+	HandlerGetSRMetrics
+	HandlerGetPeerMetrics
+	HandlerGetMetacacheListing
+	HandlerUpdateMetacacheListing
+	HandlerGetPeerBucketMetrics
+	HandlerStorageInfo
+	HandlerConsoleLog
+	HandlerListDir
+	HandlerGetLocks
+	HandlerBackgroundHealStatus
+	HandlerGetLastDayTierStats
+	HandlerSignalService
+	HandlerGetBandwidth
+	HandlerWriteAll
+	HandlerListBuckets
+	HandlerRenameDataInline
+	HandlerRenameData2
+	HandlerCheckParts2
+	HandlerRenamePart
+	HandlerClearUploadID
+	HandlerCheckParts3
+
+	// Add more above here ^^^
+	// If all handlers are used, the type of Handler can be changed.
+	// Handlers have no versioning, so non-compatible handler changes must result in new IDs.
+	handlerTest
+	handlerTest2
+	handlerLast
+)
+
+// handlerPrefixes are prefixes for handler IDs used for tracing.
+// If a handler is not listed here, it will be traced with "grid" prefix.
+var handlerPrefixes = [handlerLast]string{
+	HandlerLockLock:                    lockPrefix,
+	HandlerLockRLock:                   lockPrefix,
+	HandlerLockUnlock:                  lockPrefix,
+	HandlerLockRUnlock:                 lockPrefix,
+	HandlerLockRefresh:                 lockPrefix,
+	HandlerLockForceUnlock:             lockPrefix,
+	HandlerWalkDir:                     storagePrefix,
+	HandlerStatVol:                     storagePrefix,
+	HandlerDiskInfo:                    storagePrefix,
+	HandlerNSScanner:                   storagePrefix,
+	HandlerReadXL:                      storagePrefix,
+	HandlerReadVersion:                 storagePrefix,
+	HandlerDeleteFile:                  storagePrefix,
+	HandlerDeleteVersion:               storagePrefix,
+	HandlerUpdateMetadata:              storagePrefix,
+	HandlerWriteMetadata:               storagePrefix,
+	HandlerCheckParts:                  storagePrefix,
+	HandlerRenameData:                  storagePrefix,
+	HandlerRenameFile:                  storagePrefix,
+	HandlerReadAll:                     storagePrefix,
+	HandlerWriteAll:                    storagePrefix,
+	HandlerServerVerify:                bootstrapPrefix,
+	HandlerTrace:                       peerPrefix,
+	HandlerListen:                      peerPrefix,
+	HandlerDeleteBucketMetadata:        peerPrefix,
+	HandlerLoadBucketMetadata:          peerPrefix,
+	HandlerReloadSiteReplicationConfig: peerPrefix,
+	HandlerReloadPoolMeta:              peerPrefix,
+	HandlerStopRebalance:               peerPrefix,
+	HandlerLoadRebalanceMeta:           peerPrefix,
+	HandlerLoadTransitionTierConfig:    peerPrefix,
+	HandlerDeletePolicy:                peerPrefix,
+	HandlerLoadPolicy:                  peerPrefix,
+	HandlerLoadPolicyMapping:           peerPrefix,
+	HandlerDeleteServiceAccount:        peerPrefix,
+	HandlerLoadServiceAccount:          peerPrefix,
+	HandlerDeleteUser:                  peerPrefix,
+	HandlerLoadUser:                    peerPrefix,
+	HandlerLoadGroup:                   peerPrefix,
+	HandlerMakeBucket:                  peerPrefixS3,
+	HandlerHeadBucket:                  peerPrefixS3,
+	HandlerDeleteBucket:                peerPrefixS3,
+	HandlerHealBucket:                  healPrefix,
+	HandlerGetMetrics:                  peerPrefix,
+	HandlerGetResourceMetrics:          peerPrefix,
+	HandlerGetMemInfo:                  peerPrefix,
+	HandlerGetProcInfo:                 peerPrefix,
+	HandlerGetOSInfo:                   peerPrefix,
+	HandlerGetPartitions:               peerPrefix,
+	HandlerGetNetInfo:                  peerPrefix,
+	HandlerGetCPUs:                     peerPrefix,
+	HandlerServerInfo:                  peerPrefix,
+	HandlerGetSysConfig:                peerPrefix,
+	HandlerGetSysServices:              peerPrefix,
+	HandlerGetSysErrors:                peerPrefix,
+	HandlerGetAllBucketStats:           peerPrefix,
+	HandlerGetBucketStats:              peerPrefix,
+	HandlerGetSRMetrics:                peerPrefix,
+	HandlerGetPeerMetrics:              peerPrefix,
+	HandlerGetMetacacheListing:         peerPrefix,
+	HandlerUpdateMetacacheListing:      peerPrefix,
+	HandlerGetPeerBucketMetrics:        peerPrefix,
+	HandlerStorageInfo:                 peerPrefix,
+	HandlerConsoleLog:                  peerPrefix,
+	HandlerListDir:                     storagePrefix,
+	HandlerListBuckets:                 peerPrefixS3,
+	HandlerRenameDataInline:            storagePrefix,
+	HandlerRenameData2:                 storagePrefix,
+	HandlerCheckParts2:                 storagePrefix,
+	HandlerCheckParts3:                 storagePrefix,
+	HandlerRenamePart:                  storagePrefix,
+	HandlerClearUploadID:               peerPrefix,
+}
+
+const (
+	lockPrefix      = "lockR"
+	storagePrefix   = "storageR"
+	bootstrapPrefix = "bootstrap"
+	peerPrefix      = "peer"
+	peerPrefixS3    = "peerS3"
+	healPrefix      = "heal"
+)
+
+func init() {
+	// Static check if we exceed 255 handler ids.
+	// Extend the type to uint16 when hit.
+	if uint32(handlerLast) > 255 {
+		panic(fmt.Sprintf("out of handler IDs. %d > %d", handlerLast, 255))
+	}
+}
+
+func (h HandlerID) valid() bool {
+	return h != handlerInvalid && h < handlerLast
+}
+
+func (h HandlerID) isTestHandler() bool {
+	return h >= handlerTest && h <= handlerTest2
+}
+
+// RemoteErr is a remote error type.
+// Any error seen on a remote will be returned like this.
+type RemoteErr string
+
+// NewRemoteErr creates a new remote error.
+// The error type is not preserved.
+func NewRemoteErr(err error) *RemoteErr {
+	if err == nil {
+		return nil
+	}
+	r := RemoteErr(err.Error())
+	return &r
+}
+
+// NewRemoteErrf creates a new remote error from a format string.
+func NewRemoteErrf(format string, a ...any) *RemoteErr {
+	r := RemoteErr(fmt.Sprintf(format, a...))
+	return &r
+}
+
+// NewNPErr is a helper to no payload and optional remote error.
+// The error type is not preserved.
+func NewNPErr(err error) (NoPayload, *RemoteErr) {
+	if err == nil {
+		return NoPayload{}, nil
+	}
+	r := RemoteErr(err.Error())
+	return NoPayload{}, &r
+}
+
+// NewRemoteErrString creates a new remote error from a string.
+func NewRemoteErrString(msg string) *RemoteErr {
+	r := RemoteErr(msg)
+	return &r
+}
+
+func (r RemoteErr) Error() string {
+	return string(r)
+}
+
+// Is returns if the string representation matches.
+func (r *RemoteErr) Is(other error) bool {
+	if r == nil || other == nil {
+		return r == other
+	}
+	var o RemoteErr
+	if errors.As(other, &o) {
+		return r == &o
+	}
+	return false
+}
+
+// IsRemoteErr returns the value if the error is a RemoteErr.
+func IsRemoteErr(err error) *RemoteErr {
+	var r RemoteErr
+	if errors.As(err, &r) {
+		return &r
+	}
+	return nil
+}
+
+type (
+	// SingleHandlerFn is handlers for one to one requests.
+	// A non-nil error value will be returned as RemoteErr(msg) to client.
+	// No client information or cancellation (deadline) is available.
+	// Include this in payload if needed.
+	// Payload should be recycled with PutByteBuffer if not needed after the call.
+	SingleHandlerFn func(payload []byte) ([]byte, *RemoteErr)
+
+	// StatelessHandlerFn must handle incoming stateless request.
+	// A non-nil error value will be returned as RemoteErr(msg) to client.
+	StatelessHandlerFn func(ctx context.Context, payload []byte, resp chan<- []byte) *RemoteErr
+
+	// StatelessHandler is handlers for one to many requests,
+	// where responses may be dropped.
+	// Stateless requests provide no incoming stream and there is no flow control
+	// on outgoing messages.
+	StatelessHandler struct {
+		Handle StatelessHandlerFn
+		// OutCapacity is the output capacity on the caller.
+		// If <= 0 capacity will be 1.
+		OutCapacity int
+	}
+
+	// StreamHandlerFn must process a request with an optional initial payload.
+	// It must keep consuming from 'in' until it returns.
+	// 'in' and 'out' are independent.
+	// The handler should never close out.
+	// Buffers received from 'in'  can be recycled with PutByteBuffer.
+	// Buffers sent on out can not be referenced once sent.
+	StreamHandlerFn func(ctx context.Context, payload []byte, in <-chan []byte, out chan<- []byte) *RemoteErr
+
+	// StreamHandler handles fully bidirectional streams,
+	// There is flow control in both directions.
+	StreamHandler struct {
+		// Handle an incoming request. Initial payload is sent.
+		// Additional input packets (if any) are streamed to request.
+		// Upstream will block when request channel is full.
+		// Response packets can be sent at any time.
+		// Any non-nil error sent as response means no more responses are sent.
+		Handle StreamHandlerFn
+
+		// Subroute for handler.
+		// Subroute must be static and clients should specify a matching subroute.
+		// Should not be set unless there are different handlers for the same HandlerID.
+		Subroute string
+
+		// OutCapacity is the output capacity. If <= 0 capacity will be 1.
+		OutCapacity int
+
+		// InCapacity is the output capacity.
+		// If == 0 no input is expected
+		InCapacity int
+	}
+)
+
+type subHandlerID [32]byte
+
+func makeSubHandlerID(id HandlerID, subRoute string) subHandlerID {
+	b := subHandlerID(sha256.Sum256([]byte(subRoute)))
+	b[0] = byte(id)
+	b[1] = 0 // Reserved
+	return b
+}
+
+func (s subHandlerID) withHandler(id HandlerID) subHandlerID {
+	s[0] = byte(id)
+	s[1] = 0 // Reserved
+	return s
+}
+
+func (s *subHandlerID) String() string {
+	if s == nil {
+		return ""
+	}
+	return hex.EncodeToString(s[:])
+}
+
+func makeZeroSubHandlerID(id HandlerID) subHandlerID {
+	return subHandlerID{byte(id)}
+}
+
+type handlers struct {
+	single    [handlerLast]SingleHandlerFn
+	stateless [handlerLast]*StatelessHandler
+	streams   [handlerLast]*StreamHandler
+
+	subSingle    map[subHandlerID]SingleHandlerFn
+	subStateless map[subHandlerID]*StatelessHandler
+	subStreams   map[subHandlerID]*StreamHandler
+}
+
+func (h *handlers) init() {
+	h.subSingle = make(map[subHandlerID]SingleHandlerFn)
+	h.subStateless = make(map[subHandlerID]*StatelessHandler)
+	h.subStreams = make(map[subHandlerID]*StreamHandler)
+}
+
+func (h *handlers) hasAny(id HandlerID) bool {
+	if !id.valid() {
+		return false
+	}
+	return h.single[id] != nil || h.stateless[id] != nil || h.streams[id] != nil
+}
+
+func (h *handlers) hasSubhandler(id subHandlerID) bool {
+	return h.subSingle[id] != nil || h.subStateless[id] != nil || h.subStreams[id] != nil
+}
+
+// RoundTripper provides an interface for type roundtrip serialization.
+type RoundTripper interface {
+	msgp.Unmarshaler
+	msgp.Marshaler
+	msgp.Sizer
+
+	comparable
+}
+
+// SingleHandler is a type safe handler for single roundtrip requests.
+type SingleHandler[Req, Resp RoundTripper] struct {
+	id            HandlerID
+	sharedResp    bool
+	callReuseReq  bool
+	ignoreNilConn bool
+
+	newReq  func() Req
+	newResp func() Resp
+
+	recycleReq  func(Req)
+	recycleResp func(Resp)
+}
+
+func recycleFunc[RT RoundTripper](newRT func() RT) (newFn func() RT, recycle func(r RT)) {
+	rAny := any(newRT())
+	var rZero RT
+	if _, ok := rAny.(Recycler); ok {
+		return newRT, func(r RT) {
+			if r != rZero {
+				if rc, ok := any(r).(Recycler); ok {
+					rc.Recycle()
+				}
+			}
+		}
+	}
+	pool := bpool.Pool[RT]{
+		New: func() RT {
+			return newRT()
+		},
+	}
+	return pool.Get,
+		func(r RT) {
+			if r != rZero {
+				//nolint:staticcheck // SA6002 IT IS A GENERIC VALUE!
+				pool.Put(r)
+			}
+		}
+}
+
+// NewSingleHandler creates a typed handler that can provide Marshal/Unmarshal.
+// Use Register to register a server handler.
+// Use Call to initiate a clientside call.
+func NewSingleHandler[Req, Resp RoundTripper](h HandlerID, newReq func() Req, newResp func() Resp) *SingleHandler[Req, Resp] {
+	s := SingleHandler[Req, Resp]{id: h}
+	s.newReq, s.recycleReq = recycleFunc[Req](newReq)
+	s.newResp, s.recycleResp = recycleFunc[Resp](newResp)
+	if _, ok := any(newReq()).(Recycler); ok {
+		s.callReuseReq = true
+	}
+	return &s
+}
+
+// PutResponse will accept a response for reuse.
+// This can be used by a caller to recycle a response after receiving it from a Call.
+func (h *SingleHandler[Req, Resp]) PutResponse(r Resp) {
+	h.recycleResp(r)
+}
+
+// AllowCallRequestPool indicates it is safe to reuse the request
+// on the client side, meaning the request is recycled/pooled when a request is sent.
+// CAREFUL: This should only be used when there are no pointers, slices that aren't freshly constructed.
+func (h *SingleHandler[Req, Resp]) AllowCallRequestPool(b bool) *SingleHandler[Req, Resp] {
+	h.callReuseReq = b
+	return h
+}
+
+// IgnoreNilConn will ignore nil connections when calling.
+// This will make Call return nil instead of ErrDisconnected when the connection is nil.
+// This may only be set ONCE before use.
+func (h *SingleHandler[Req, Resp]) IgnoreNilConn() *SingleHandler[Req, Resp] {
+	if h.ignoreNilConn {
+		gridLogOnceIf(context.Background(), fmt.Errorf("%s: IgnoreNilConn called twice", h.id.String()), h.id.String()+"IgnoreNilConn")
+	}
+	h.ignoreNilConn = true
+	return h
+}
+
+// WithSharedResponse indicates it is unsafe to reuse the response
+// when it has been returned on a handler.
+// This will disable automatic response recycling/pooling.
+// Typically this is used when the response sharing part of its data structure.
+func (h *SingleHandler[Req, Resp]) WithSharedResponse() *SingleHandler[Req, Resp] {
+	h.sharedResp = true
+	return h
+}
+
+// NewResponse creates a new response.
+// Handlers can use this to create a reusable response.
+// The response may be reused, so caller should clear any fields.
+func (h *SingleHandler[Req, Resp]) NewResponse() Resp {
+	return h.newResp()
+}
+
+// NewRequest creates a new request.
+// Handlers can use this to create a reusable request.
+// The request may be reused, so caller should clear any fields.
+func (h *SingleHandler[Req, Resp]) NewRequest() Req {
+	return h.newReq()
+}
+
+// Register a handler for a Req -> Resp roundtrip.
+// Requests are automatically recycled.
+func (h *SingleHandler[Req, Resp]) Register(m *Manager, handle func(req Req) (resp Resp, err *RemoteErr), subroute ...string) error {
+	if h.newReq == nil {
+		return errors.New("newReq nil in NewSingleHandler")
+	}
+	if h.newResp == nil {
+		return errors.New("newResp nil in NewSingleHandler")
+	}
+	return m.RegisterSingleHandler(h.id, func(payload []byte) ([]byte, *RemoteErr) {
+		req := h.NewRequest()
+		_, err := req.UnmarshalMsg(payload)
+		if err != nil {
+			PutByteBuffer(payload)
+			r := RemoteErr(err.Error())
+			return nil, &r
+		}
+		resp, rerr := handle(req)
+		h.recycleReq(req)
+
+		if rerr != nil {
+			PutByteBuffer(payload)
+			return nil, rerr
+		}
+		payload, err = resp.MarshalMsg(payload[:0])
+		if !h.sharedResp {
+			h.PutResponse(resp)
+		}
+		if err != nil {
+			PutByteBuffer(payload)
+			r := RemoteErr(err.Error())
+			return nil, &r
+		}
+		return payload, nil
+	}, subroute...)
+}
+
+// Requester is able to send requests to a remote.
+type Requester interface {
+	Request(ctx context.Context, h HandlerID, req []byte) ([]byte, error)
+}
+
+// Call the remote with the request and return the response.
+// The response should be returned with PutResponse when no error.
+// If no deadline is set, a 1-minute deadline is added.
+func (h *SingleHandler[Req, Resp]) Call(ctx context.Context, c Requester, req Req) (resp Resp, err error) {
+	if c == nil {
+		if h.ignoreNilConn {
+			return resp, nil
+		}
+		return resp, ErrDisconnected
+	}
+	payload, err := req.MarshalMsg(GetByteBufferCap(req.Msgsize()))
+	if err != nil {
+		return resp, err
+	}
+	switch any(req).(type) {
+	case *MSS, *URLValues:
+		ctx = context.WithValue(ctx, TraceParamsKey{}, req)
+	case *NoPayload, *Bytes:
+		// do not need to trace nopayload and bytes payload
+	default:
+		ctx = context.WithValue(ctx, TraceParamsKey{}, fmt.Sprintf("type=%T", req))
+	}
+	if h.callReuseReq {
+		defer h.recycleReq(req)
+	}
+	res, err := c.Request(ctx, h.id, payload)
+	PutByteBuffer(payload)
+	if err != nil {
+		return resp, err
+	}
+	defer PutByteBuffer(res)
+	r := h.NewResponse()
+	_, err = r.UnmarshalMsg(res)
+	if err != nil {
+		h.PutResponse(r)
+		return resp, err
+	}
+	return r, err
+}
+
+// RemoteClient contains information about the caller.
+type RemoteClient struct {
+	Name string
+}
+
+type (
+	ctxCallerKey   = struct{}
+	ctxSubrouteKey = struct{}
+)
+
+// GetCaller returns caller information from contexts provided to handlers.
+func GetCaller(ctx context.Context) *RemoteClient {
+	val, _ := ctx.Value(ctxCallerKey{}).(*RemoteClient)
+	return val
+}
+
+// GetSubroute returns caller information from contexts provided to handlers.
+func GetSubroute(ctx context.Context) string {
+	//nolint:staticcheck // SA1029 Staticcheck is drunk.
+	val, _ := ctx.Value(ctxSubrouteKey{}).(string)
+	return val
+}
+
+func setCaller(ctx context.Context, cl *RemoteClient) context.Context {
+	//nolint:staticcheck // SA1029 Staticcheck is drunk.
+	return context.WithValue(ctx, ctxCallerKey{}, cl)
+}
+
+func setSubroute(ctx context.Context, s string) context.Context {
+	//nolint:staticcheck // SA1029 Staticcheck is drunk.
+	return context.WithValue(ctx, ctxSubrouteKey{}, s)
+}
+
+// StreamTypeHandler is a type safe handler for streaming requests.
+type StreamTypeHandler[Payload, Req, Resp RoundTripper] struct {
+	WithPayload bool
+
+	// Override the default capacities (1)
+	OutCapacity int
+
+	// Set to 0 if no input is expected.
+	// Will be 0 if newReq is nil.
+	InCapacity int
+
+	reqPool        bpool.Pool[Req]
+	respPool       bpool.Pool[Resp]
+	id             HandlerID
+	newPayload     func() Payload
+	nilReq         Req
+	nilResp        Resp
+	sharedResponse bool
+}
+
+// NewStream creates a typed handler that can provide Marshal/Unmarshal.
+// Use Register to register a server handler.
+// Use Call to initiate a clientside call.
+// newPayload can be nil. In that case payloads will always be nil.
+// newReq can be nil. In that case no input stream is expected and the handler will be called with nil 'in' channel.
+func NewStream[Payload, Req, Resp RoundTripper](h HandlerID, newPayload func() Payload, newReq func() Req, newResp func() Resp) *StreamTypeHandler[Payload, Req, Resp] {
+	if newResp == nil {
+		panic("newResp missing in NewStream")
+	}
+
+	s := newStreamHandler[Payload, Req, Resp](h)
+	if newReq != nil {
+		s.reqPool.New = func() Req {
+			return newReq()
+		}
+	} else {
+		s.InCapacity = 0
+	}
+	s.respPool.New = func() Resp {
+		return newResp()
+	}
+	s.newPayload = newPayload
+	s.WithPayload = newPayload != nil
+	return s
+}
+
+// WithSharedResponse indicates it is unsafe to reuse the response.
+// Typically this is used when the response sharing part of its data structure.
+func (h *StreamTypeHandler[Payload, Req, Resp]) WithSharedResponse() *StreamTypeHandler[Payload, Req, Resp] {
+	h.sharedResponse = true
+	return h
+}
+
+// NewPayload creates a new payload.
+func (h *StreamTypeHandler[Payload, Req, Resp]) NewPayload() Payload {
+	return h.newPayload()
+}
+
+// NewRequest creates a new request.
+// The struct may be reused, so caller should clear any fields.
+func (h *StreamTypeHandler[Payload, Req, Resp]) NewRequest() Req {
+	return h.reqPool.Get()
+}
+
+// PutRequest will accept a request for reuse.
+// These should be returned by the handler.
+func (h *StreamTypeHandler[Payload, Req, Resp]) PutRequest(r Req) {
+	if r != h.nilReq {
+		//nolint:staticcheck // SA6002 IT IS A GENERIC VALUE! (and always a pointer)
+		h.reqPool.Put(r)
+	}
+}
+
+// PutResponse will accept a response for reuse.
+// These should be returned by the caller.
+func (h *StreamTypeHandler[Payload, Req, Resp]) PutResponse(r Resp) {
+	if r != h.nilResp {
+		//nolint:staticcheck // SA6002 IT IS A GENERIC VALUE! (and always a pointer)
+		h.respPool.Put(r)
+	}
+}
+
+// NewResponse creates a new response.
+// Handlers can use this to create a reusable response.
+func (h *StreamTypeHandler[Payload, Req, Resp]) NewResponse() Resp {
+	return h.respPool.Get()
+}
+
+func newStreamHandler[Payload, Req, Resp RoundTripper](h HandlerID) *StreamTypeHandler[Payload, Req, Resp] {
+	return &StreamTypeHandler[Payload, Req, Resp]{id: h, InCapacity: 1, OutCapacity: 1}
+}
+
+// Register a handler for two-way streaming with payload, input stream and output stream.
+// An optional subroute can be given. Multiple entries are joined with '/'.
+func (h *StreamTypeHandler[Payload, Req, Resp]) Register(m *Manager, handle func(ctx context.Context, p Payload, in <-chan Req, out chan<- Resp) *RemoteErr, subroute ...string) error {
+	return h.register(m, handle, subroute...)
+}
+
+// WithOutCapacity adjusts the output capacity from the handler perspective.
+// This must be done prior to registering the handler.
+func (h *StreamTypeHandler[Payload, Req, Resp]) WithOutCapacity(out int) *StreamTypeHandler[Payload, Req, Resp] {
+	h.OutCapacity = out
+	return h
+}
+
+// WithInCapacity adjusts the input capacity from the handler perspective.
+// This must be done prior to registering the handler.
+func (h *StreamTypeHandler[Payload, Req, Resp]) WithInCapacity(in int) *StreamTypeHandler[Payload, Req, Resp] {
+	h.InCapacity = in
+	return h
+}
+
+// RegisterNoInput a handler for one-way streaming with payload and output stream.
+// An optional subroute can be given. Multiple entries are joined with '/'.
+func (h *StreamTypeHandler[Payload, Req, Resp]) RegisterNoInput(m *Manager, handle func(ctx context.Context, p Payload, out chan<- Resp) *RemoteErr, subroute ...string) error {
+	h.InCapacity = 0
+	return h.register(m, func(ctx context.Context, p Payload, in <-chan Req, out chan<- Resp) *RemoteErr {
+		return handle(ctx, p, out)
+	}, subroute...)
+}
+
+// RegisterNoPayload a handler for one-way streaming with payload and output stream.
+// An optional subroute can be given. Multiple entries are joined with '/'.
+func (h *StreamTypeHandler[Payload, Req, Resp]) RegisterNoPayload(m *Manager, handle func(ctx context.Context, in <-chan Req, out chan<- Resp) *RemoteErr, subroute ...string) error {
+	h.WithPayload = false
+	return h.register(m, func(ctx context.Context, p Payload, in <-chan Req, out chan<- Resp) *RemoteErr {
+		return handle(ctx, in, out)
+	}, subroute...)
+}
+
+// Register a handler for two-way streaming with optional payload and input stream.
+func (h *StreamTypeHandler[Payload, Req, Resp]) register(m *Manager, handle func(ctx context.Context, p Payload, in <-chan Req, out chan<- Resp) *RemoteErr, subroute ...string) error {
+	return m.RegisterStreamingHandler(h.id, StreamHandler{
+		Handle: func(ctx context.Context, payload []byte, in <-chan []byte, out chan<- []byte) *RemoteErr {
+			var plT Payload
+			if h.WithPayload {
+				plT = h.NewPayload()
+				_, err := plT.UnmarshalMsg(payload)
+				PutByteBuffer(payload)
+				if err != nil {
+					r := RemoteErr(err.Error())
+					return &r
+				}
+			}
+
+			var inT chan Req
+			if h.InCapacity > 0 {
+				// Don't add extra buffering
+				inT = make(chan Req)
+				go func() {
+					defer xioutil.SafeClose(inT)
+					for {
+						select {
+						case <-ctx.Done():
+							return
+						case v, ok := <-in:
+							if !ok {
+								return
+							}
+							input := h.NewRequest()
+							_, err := input.UnmarshalMsg(v)
+							if err != nil {
+								gridLogOnceIf(ctx, err, err.Error())
+							}
+							PutByteBuffer(v)
+							// Send input
+							select {
+							case <-ctx.Done():
+								return
+							case inT <- input:
+							}
+						}
+					}
+				}()
+			}
+			outT := make(chan Resp)
+			outDone := make(chan struct{})
+			go func() {
+				defer xioutil.SafeClose(outDone)
+				dropOutput := false
+				for v := range outT {
+					if dropOutput {
+						continue
+					}
+					dst, err := v.MarshalMsg(GetByteBufferCap(v.Msgsize()))
+					if err != nil {
+						gridLogOnceIf(ctx, err, err.Error())
+					}
+					if !h.sharedResponse {
+						h.PutResponse(v)
+					}
+					select {
+					case <-ctx.Done():
+						dropOutput = true
+					case out <- dst:
+					}
+				}
+			}()
+			rErr := handle(ctx, plT, inT, outT)
+			xioutil.SafeClose(outT)
+			<-outDone
+			return rErr
+		}, OutCapacity: h.OutCapacity, InCapacity: h.InCapacity, Subroute: strings.Join(subroute, "/"),
+	})
+}
+
+// TypedStream is a stream with specific types.
+type TypedStream[Req, Resp RoundTripper] struct {
+	// responses from the remote server.
+	// Channel will be closed after error or when remote closes.
+	// responses *must* be read to either an error is returned or the channel is closed.
+	responses *Stream
+	newResp   func() Resp
+
+	// Requests sent to the server.
+	// If the handler is defined with 0 incoming capacity this will be nil.
+	// Channel *must* be closed to signal the end of the stream.
+	// If the request context is canceled, the stream will no longer process requests.
+	Requests chan<- Req
+}
+
+// Results returns the results from the remote server one by one.
+// If any error is returned by the callback, the stream will be canceled.
+// If the context is canceled, the stream will be canceled.
+func (s *TypedStream[Req, Resp]) Results(next func(resp Resp) error) (err error) {
+	return s.responses.Results(func(b []byte) error {
+		resp := s.newResp()
+		_, err := resp.UnmarshalMsg(b)
+		if err != nil {
+			return err
+		}
+		return next(resp)
+	})
+}
+
+// Streamer creates a stream.
+type Streamer interface {
+	NewStream(ctx context.Context, h HandlerID, payload []byte) (st *Stream, err error)
+}
+
+// Call the remove with the request and
+func (h *StreamTypeHandler[Payload, Req, Resp]) Call(ctx context.Context, c Streamer, payload Payload) (st *TypedStream[Req, Resp], err error) {
+	if c == nil {
+		return nil, ErrDisconnected
+	}
+	var payloadB []byte
+	if h.WithPayload {
+		var err error
+		payloadB, err = payload.MarshalMsg(GetByteBufferCap(payload.Msgsize()))
+		if err != nil {
+			return nil, err
+		}
+	}
+	stream, err := c.NewStream(ctx, h.id, payloadB)
+	PutByteBuffer(payloadB)
+	if err != nil {
+		return nil, err
+	}
+
+	// respT := make(chan TypedResponse[Resp])
+	var reqT chan Req
+	if h.InCapacity > 0 {
+		reqT = make(chan Req)
+		// Request handler
+		if stream.Requests == nil {
+			return nil, fmt.Errorf("internal error: stream request channel nil")
+		}
+		go func() {
+			defer xioutil.SafeClose(stream.Requests)
+			for req := range reqT {
+				b, err := req.MarshalMsg(GetByteBufferCap(req.Msgsize()))
+				if err != nil {
+					gridLogOnceIf(ctx, err, err.Error())
+				}
+				h.PutRequest(req)
+				stream.Requests <- b
+			}
+		}()
+	} else if stream.Requests != nil {
+		xioutil.SafeClose(stream.Requests)
+	}
+
+	return &TypedStream[Req, Resp]{responses: stream, newResp: h.NewResponse, Requests: reqT}, nil
+}

internal/grid/handlers_string.go

@@ -0,0 +1,103 @@
+// Code generated by "stringer -type=HandlerID -output=handlers_string.go -trimprefix=Handler msg.go handlers.go"; DO NOT EDIT.
+
+package grid
+
+import "strconv"
+
+func _() {
+	// An "invalid array index" compiler error signifies that the constant values have changed.
+	// Re-run the stringer command to generate them again.
+	var x [1]struct{}
+	_ = x[handlerInvalid-0]
+	_ = x[HandlerLockLock-1]
+	_ = x[HandlerLockRLock-2]
+	_ = x[HandlerLockUnlock-3]
+	_ = x[HandlerLockRUnlock-4]
+	_ = x[HandlerLockRefresh-5]
+	_ = x[HandlerLockForceUnlock-6]
+	_ = x[HandlerWalkDir-7]
+	_ = x[HandlerStatVol-8]
+	_ = x[HandlerDiskInfo-9]
+	_ = x[HandlerNSScanner-10]
+	_ = x[HandlerReadXL-11]
+	_ = x[HandlerReadVersion-12]
+	_ = x[HandlerDeleteFile-13]
+	_ = x[HandlerDeleteVersion-14]
+	_ = x[HandlerUpdateMetadata-15]
+	_ = x[HandlerWriteMetadata-16]
+	_ = x[HandlerCheckParts-17]
+	_ = x[HandlerRenameData-18]
+	_ = x[HandlerRenameFile-19]
+	_ = x[HandlerReadAll-20]
+	_ = x[HandlerServerVerify-21]
+	_ = x[HandlerTrace-22]
+	_ = x[HandlerListen-23]
+	_ = x[HandlerDeleteBucketMetadata-24]
+	_ = x[HandlerLoadBucketMetadata-25]
+	_ = x[HandlerReloadSiteReplicationConfig-26]
+	_ = x[HandlerReloadPoolMeta-27]
+	_ = x[HandlerStopRebalance-28]
+	_ = x[HandlerLoadRebalanceMeta-29]
+	_ = x[HandlerLoadTransitionTierConfig-30]
+	_ = x[HandlerDeletePolicy-31]
+	_ = x[HandlerLoadPolicy-32]
+	_ = x[HandlerLoadPolicyMapping-33]
+	_ = x[HandlerDeleteServiceAccount-34]
+	_ = x[HandlerLoadServiceAccount-35]
+	_ = x[HandlerDeleteUser-36]
+	_ = x[HandlerLoadUser-37]
+	_ = x[HandlerLoadGroup-38]
+	_ = x[HandlerHealBucket-39]
+	_ = x[HandlerMakeBucket-40]
+	_ = x[HandlerHeadBucket-41]
+	_ = x[HandlerDeleteBucket-42]
+	_ = x[HandlerGetMetrics-43]
+	_ = x[HandlerGetResourceMetrics-44]
+	_ = x[HandlerGetMemInfo-45]
+	_ = x[HandlerGetProcInfo-46]
+	_ = x[HandlerGetOSInfo-47]
+	_ = x[HandlerGetPartitions-48]
+	_ = x[HandlerGetNetInfo-49]
+	_ = x[HandlerGetCPUs-50]
+	_ = x[HandlerServerInfo-51]
+	_ = x[HandlerGetSysConfig-52]
+	_ = x[HandlerGetSysServices-53]
+	_ = x[HandlerGetSysErrors-54]
+	_ = x[HandlerGetAllBucketStats-55]
+	_ = x[HandlerGetBucketStats-56]
+	_ = x[HandlerGetSRMetrics-57]
+	_ = x[HandlerGetPeerMetrics-58]
+	_ = x[HandlerGetMetacacheListing-59]
+	_ = x[HandlerUpdateMetacacheListing-60]
+	_ = x[HandlerGetPeerBucketMetrics-61]
+	_ = x[HandlerStorageInfo-62]
+	_ = x[HandlerConsoleLog-63]
+	_ = x[HandlerListDir-64]
+	_ = x[HandlerGetLocks-65]
+	_ = x[HandlerBackgroundHealStatus-66]
+	_ = x[HandlerGetLastDayTierStats-67]
+	_ = x[HandlerSignalService-68]
+	_ = x[HandlerGetBandwidth-69]
+	_ = x[HandlerWriteAll-70]
+	_ = x[HandlerListBuckets-71]
+	_ = x[HandlerRenameDataInline-72]
+	_ = x[HandlerRenameData2-73]
+	_ = x[HandlerCheckParts2-74]
+	_ = x[HandlerRenamePart-75]
+	_ = x[HandlerClearUploadID-76]
+	_ = x[HandlerCheckParts3-77]
+	_ = x[handlerTest-78]
+	_ = x[handlerTest2-79]
+	_ = x[handlerLast-80]
+}
+
+const _HandlerID_name = "handlerInvalidLockLockLockRLockLockUnlockLockRUnlockLockRefreshLockForceUnlockWalkDirStatVolDiskInfoNSScannerReadXLReadVersionDeleteFileDeleteVersionUpdateMetadataWriteMetadataCheckPartsRenameDataRenameFileReadAllServerVerifyTraceListenDeleteBucketMetadataLoadBucketMetadataReloadSiteReplicationConfigReloadPoolMetaStopRebalanceLoadRebalanceMetaLoadTransitionTierConfigDeletePolicyLoadPolicyLoadPolicyMappingDeleteServiceAccountLoadServiceAccountDeleteUserLoadUserLoadGroupHealBucketMakeBucketHeadBucketDeleteBucketGetMetricsGetResourceMetricsGetMemInfoGetProcInfoGetOSInfoGetPartitionsGetNetInfoGetCPUsServerInfoGetSysConfigGetSysServicesGetSysErrorsGetAllBucketStatsGetBucketStatsGetSRMetricsGetPeerMetricsGetMetacacheListingUpdateMetacacheListingGetPeerBucketMetricsStorageInfoConsoleLogListDirGetLocksBackgroundHealStatusGetLastDayTierStatsSignalServiceGetBandwidthWriteAllListBucketsRenameDataInlineRenameData2CheckParts2RenamePartClearUploadIDCheckParts3handlerTesthandlerTest2handlerLast"
+
+var _HandlerID_index = [...]uint16{0, 14, 22, 31, 41, 52, 63, 78, 85, 92, 100, 109, 115, 126, 136, 149, 163, 176, 186, 196, 206, 213, 225, 230, 236, 256, 274, 301, 315, 328, 345, 369, 381, 391, 408, 428, 446, 456, 464, 473, 483, 493, 503, 515, 525, 543, 553, 564, 573, 586, 596, 603, 613, 625, 639, 651, 668, 682, 694, 708, 727, 749, 769, 780, 790, 797, 805, 825, 844, 857, 869, 877, 888, 904, 915, 926, 936, 949, 960, 971, 983, 994}
+
+func (i HandlerID) String() string {
+	if i >= HandlerID(len(_HandlerID_index)-1) {
+		return "HandlerID(" + strconv.FormatInt(int64(i), 10) + ")"
+	}
+	return _HandlerID_name[_HandlerID_index[i]:_HandlerID_index[i+1]]
+}

internal/grid/manager.go

@@ -0,0 +1,385 @@
+// Copyright (c) 2015-2023 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package grid
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"runtime/debug"
+	"strings"
+	"time"
+
+	"github.com/gobwas/ws"
+	"github.com/gobwas/ws/wsutil"
+	"github.com/google/uuid"
+	"github.com/minio/madmin-go/v3"
+	"github.com/minio/minio/internal/deadlineconn"
+	"github.com/minio/minio/internal/pubsub"
+	"github.com/minio/mux"
+)
+
+const (
+	// apiVersion is a major version of the entire api.
+	// Bumping this should only be done when overall,
+	// incompatible changes are made, not when adding a new handler
+	// or changing an existing handler.
+	apiVersion = "v1"
+
+	// RoutePath is the remote path to connect to.
+	RoutePath = "/minio/grid/" + apiVersion
+
+	// RouteLockPath is the remote lock path to connect to.
+	RouteLockPath = "/minio/grid/lock/" + apiVersion
+)
+
+// Manager will contain all the connections to the grid.
+// It also handles incoming requests and routes them to the appropriate connection.
+type Manager struct {
+	// ID is an instance ID, that will change whenever the server restarts.
+	// This allows remotes to keep track of whether state is preserved.
+	ID uuid.UUID
+
+	// Immutable after creation, so no locks.
+	targets map[string]*Connection
+
+	// serverside handlers.
+	handlers handlers
+
+	// local host name.
+	local string
+
+	// authToken is a function that will validate a token.
+	authToken ValidateTokenFn
+
+	// routePath indicates the dial route path
+	routePath string
+}
+
+// ManagerOptions are options for creating a new grid manager.
+type ManagerOptions struct {
+	Local        string        // Local host name.
+	Hosts        []string      // All hosts, including local in the grid.
+	Incoming     func(n int64) // Record incoming bytes.
+	Outgoing     func(n int64) // Record outgoing bytes.
+	BlockConnect chan struct{} // If set, incoming and outgoing connections will be blocked until closed.
+	RoutePath    string
+	TraceTo      *pubsub.PubSub[madmin.TraceInfo, madmin.TraceType]
+	Dialer       ConnDialer
+	// Sign a token for the given audience.
+	AuthFn AuthFn
+	// Callbacks to validate incoming connections.
+	AuthToken ValidateTokenFn
+}
+
+// NewManager creates a new grid manager
+func NewManager(ctx context.Context, o ManagerOptions) (*Manager, error) {
+	found := false
+	if o.AuthToken == nil {
+		return nil, fmt.Errorf("grid: AuthToken not set")
+	}
+	if o.Dialer == nil {
+		return nil, fmt.Errorf("grid: Dialer not set")
+	}
+	if o.AuthFn == nil {
+		return nil, fmt.Errorf("grid: AuthFn not set")
+	}
+	m := &Manager{
+		ID:        uuid.New(),
+		targets:   make(map[string]*Connection, len(o.Hosts)),
+		local:     o.Local,
+		authToken: o.AuthToken,
+		routePath: o.RoutePath,
+	}
+	m.handlers.init()
+	if ctx == nil {
+		ctx = context.Background()
+	}
+
+	for _, host := range o.Hosts {
+		if host == o.Local {
+			if found {
+				return nil, fmt.Errorf("grid: local host found multiple times")
+			}
+			found = true
+			// No connection to local.
+			continue
+		}
+		m.targets[host] = newConnection(connectionParams{
+			ctx:           ctx,
+			id:            m.ID,
+			local:         o.Local,
+			remote:        host,
+			handlers:      &m.handlers,
+			blockConnect:  o.BlockConnect,
+			publisher:     o.TraceTo,
+			incomingBytes: o.Incoming,
+			outgoingBytes: o.Outgoing,
+			dialer:        o.Dialer,
+			authFn:        o.AuthFn,
+		})
+	}
+	if !found {
+		return nil, fmt.Errorf("grid: local host (%s) not found in cluster setup", o.Local)
+	}
+
+	return m, nil
+}
+
+// AddToMux will add the grid manager to the given mux.
+func (m *Manager) AddToMux(router *mux.Router, authReq func(r *http.Request) error) {
+	router.Handle(m.routePath, m.Handler(authReq))
+}
+
+// Handler returns a handler that can be used to serve grid requests.
+// This should be connected on RoutePath to the main server.
+func (m *Manager) Handler(authReq func(r *http.Request) error) http.HandlerFunc {
+	return func(w http.ResponseWriter, req *http.Request) {
+		defer func() {
+			if debugPrint {
+				fmt.Printf("grid: Handler returning from: %v %v\n", req.Method, req.URL)
+			}
+			if r := recover(); r != nil {
+				debug.PrintStack()
+				err := fmt.Errorf("grid: panic: %v\n", r)
+				gridLogIf(context.Background(), err, err.Error())
+				w.WriteHeader(http.StatusInternalServerError)
+			}
+		}()
+		if debugPrint {
+			fmt.Printf("grid: Got a %s request for: %v\n", req.Method, req.URL)
+		}
+		ctx := req.Context()
+		if err := authReq(req); err != nil {
+			gridLogOnceIf(ctx, fmt.Errorf("auth %s: %w", req.RemoteAddr, err), req.RemoteAddr)
+			w.WriteHeader(http.StatusForbidden)
+			return
+		}
+		conn, _, _, err := ws.UpgradeHTTP(req, w)
+		if err != nil {
+			if debugPrint {
+				fmt.Printf("grid: Unable to upgrade: %v. http.ResponseWriter is type %T\n", err, w)
+			}
+			w.WriteHeader(http.StatusUpgradeRequired)
+			return
+		}
+		m.IncomingConn(ctx, conn)
+	}
+}
+
+// IncomingConn will handle an incoming connection.
+// This should be called with the incoming connection after accept.
+// Auth is handled internally, as well as disconnecting any connections from the same host.
+func (m *Manager) IncomingConn(ctx context.Context, conn net.Conn) {
+	// We manage our own deadlines.
+	conn = deadlineconn.Unwrap(conn)
+	remoteAddr := conn.RemoteAddr().String()
+	// will write an OpConnectResponse message to the remote and log it once locally.
+	defer conn.Close()
+	writeErr := func(err error) {
+		if err == nil {
+			return
+		}
+		if errors.Is(err, io.EOF) {
+			return
+		}
+		gridLogOnceIf(ctx, err, remoteAddr)
+		resp := connectResp{
+			ID:             m.ID,
+			Accepted:       false,
+			RejectedReason: err.Error(),
+		}
+		if b, err := resp.MarshalMsg(nil); err == nil {
+			msg := message{
+				Op:      OpConnectResponse,
+				Payload: b,
+			}
+			if b, err := msg.MarshalMsg(nil); err == nil {
+				wsutil.WriteMessage(conn, ws.StateServerSide, ws.OpBinary, b)
+			}
+		}
+	}
+	defer conn.Close()
+	if debugPrint {
+		fmt.Printf("grid: Upgraded request: %v\n", remoteAddr)
+	}
+
+	msg, _, err := wsutil.ReadClientData(conn)
+	if err != nil {
+		writeErr(fmt.Errorf("reading connect: %w", err))
+		return
+	}
+	if debugPrint {
+		fmt.Printf("%s handler: Got message, length %v\n", m.local, len(msg))
+	}
+
+	var message message
+	_, _, err = message.parse(msg)
+	if err != nil {
+		writeErr(fmt.Errorf("error parsing grid connect: %w", err))
+		return
+	}
+	if message.Op != OpConnect {
+		writeErr(fmt.Errorf("unexpected connect op: %v", message.Op))
+		return
+	}
+	var cReq connectReq
+	_, err = cReq.UnmarshalMsg(message.Payload)
+	if err != nil {
+		writeErr(fmt.Errorf("error parsing connectReq: %w", err))
+		return
+	}
+	remote := m.targets[cReq.Host]
+	if remote == nil {
+		writeErr(fmt.Errorf("unknown incoming host: %v", cReq.Host))
+		return
+	}
+	if time.Since(cReq.Time).Abs() > 5*time.Minute {
+		writeErr(fmt.Errorf("time difference too large between servers: %v", time.Since(cReq.Time).Abs()))
+		return
+	}
+	if err := m.authToken(cReq.Token); err != nil {
+		writeErr(fmt.Errorf("auth token: %w", err))
+		return
+	}
+
+	if debugPrint {
+		fmt.Printf("handler: Got Connect Req %+v\n", cReq)
+	}
+	writeErr(remote.handleIncoming(ctx, conn, cReq))
+}
+
+// AuthFn should provide an authentication string for the given aud.
+type AuthFn func() string
+
+// ValidateAuthFn should check authentication for the given aud.
+type ValidateAuthFn func(auth string) string
+
+// Connection will return the connection for the specified host.
+// If the host does not exist nil will be returned.
+func (m *Manager) Connection(host string) *Connection {
+	return m.targets[host]
+}
+
+// RegisterSingleHandler will register a stateless handler that serves
+// []byte -> ([]byte, error) requests.
+// subroutes are joined with "/" to a single subroute.
+func (m *Manager) RegisterSingleHandler(id HandlerID, h SingleHandlerFn, subroute ...string) error {
+	if !id.valid() {
+		return ErrUnknownHandler
+	}
+	s := strings.Join(subroute, "/")
+	if debugPrint {
+		fmt.Println("RegisterSingleHandler: ", id.String(), "subroute:", s)
+	}
+
+	if len(subroute) == 0 {
+		if m.handlers.hasAny(id) && !id.isTestHandler() {
+			return fmt.Errorf("handler %v: %w", id.String(), ErrHandlerAlreadyExists)
+		}
+
+		m.handlers.single[id] = h
+		return nil
+	}
+	subID := makeSubHandlerID(id, s)
+	if m.handlers.hasSubhandler(subID) && !id.isTestHandler() {
+		return fmt.Errorf("handler %v, subroute:%v: %w", id.String(), s, ErrHandlerAlreadyExists)
+	}
+	m.handlers.subSingle[subID] = h
+	// Copy so clients can also pick it up for other subpaths.
+	m.handlers.subSingle[makeZeroSubHandlerID(id)] = h
+	return nil
+}
+
+/*
+// RegisterStateless will register a stateless handler that serves
+// []byte -> stream of ([]byte, error) requests.
+func (m *Manager) RegisterStateless(id HandlerID, h StatelessHandler) error {
+	if !id.valid() {
+		return ErrUnknownHandler
+	}
+	if m.handlers.hasAny(id) && !id.isTestHandler() {
+		return ErrHandlerAlreadyExists
+	}
+
+	m.handlers.stateless[id] = &h
+	return nil
+}
+*/
+
+// RegisterStreamingHandler will register a stateless handler that serves
+// two-way streaming requests.
+func (m *Manager) RegisterStreamingHandler(id HandlerID, h StreamHandler) error {
+	if !id.valid() {
+		return ErrUnknownHandler
+	}
+	if debugPrint {
+		fmt.Println("RegisterStreamingHandler: subroute:", h.Subroute)
+	}
+	if h.Subroute == "" {
+		if m.handlers.hasAny(id) && !id.isTestHandler() {
+			return ErrHandlerAlreadyExists
+		}
+		m.handlers.streams[id] = &h
+		return nil
+	}
+	subID := makeSubHandlerID(id, h.Subroute)
+	if m.handlers.hasSubhandler(subID) && !id.isTestHandler() {
+		return ErrHandlerAlreadyExists
+	}
+	m.handlers.subStreams[subID] = &h
+	// Copy so clients can also pick it up for other subpaths.
+	m.handlers.subStreams[makeZeroSubHandlerID(id)] = &h
+	return nil
+}
+
+// HostName returns the name of the local host.
+func (m *Manager) HostName() string {
+	return m.local
+}
+
+// Targets returns the names of all remote targets.
+func (m *Manager) Targets() []string {
+	var res []string
+	for k := range m.targets {
+		res = append(res, k)
+	}
+	return res
+}
+
+// debugMsg should *only* be used by tests.
+//
+//lint:ignore U1000 This is used by tests.
+func (m *Manager) debugMsg(d debugMsg, args ...any) {
+	for _, c := range m.targets {
+		c.debugMsg(d, args...)
+	}
+}
+
+// ConnStats returns the connection statistics for all connections.
+func (m *Manager) ConnStats() madmin.RPCMetrics {
+	var res madmin.RPCMetrics
+	for _, c := range m.targets {
+		t := c.Stats()
+		res.Merge(&t)
+	}
+	return res
+}

internal/grid/msg.go

@@ -0,0 +1,308 @@
+// Copyright (c) 2015-2023 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package grid
+
+import (
+	"encoding/binary"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/tinylib/msgp/msgp"
+	"github.com/zeebo/xxh3"
+)
+
+// Op is operation type.
+//
+//go:generate msgp -unexported -file=$GOFILE
+//go:generate stringer -type=Op -output=msg_string.go -trimprefix=Op $GOFILE
+
+// Op is operation type messages.
+type Op uint8
+
+// HandlerID is the ID for the handler of a specific type.
+type HandlerID uint8
+
+const (
+	// OpConnect is a connect request.
+	OpConnect Op = iota + 1
+
+	// OpConnectResponse is a response to a connect request.
+	OpConnectResponse
+
+	// OpPing is a ping request.
+	// If a mux id is specified that mux is pinged.
+	// Clients send ping requests.
+	OpPing
+
+	// OpPong is a OpPing response returned by the server.
+	OpPong
+
+	// OpConnectMux will connect a new mux with optional payload.
+	OpConnectMux
+
+	// OpMuxConnectError is an  error while connecting a mux.
+	OpMuxConnectError
+
+	// OpDisconnectClientMux instructs a client to disconnect a mux
+	OpDisconnectClientMux
+
+	// OpDisconnectServerMux instructs a server to disconnect (cancel) a server mux
+	OpDisconnectServerMux
+
+	// OpMuxClientMsg contains a message to a client Mux
+	OpMuxClientMsg
+
+	// OpMuxServerMsg contains a message to a server Mux
+	OpMuxServerMsg
+
+	// OpUnblockSrvMux contains a message that a server mux is unblocked with one.
+	// Only Stateful streams has flow control.
+	OpUnblockSrvMux
+
+	// OpUnblockClMux contains a message that a client mux is unblocked with one.
+	// Only Stateful streams has flow control.
+	OpUnblockClMux
+
+	// OpAckMux acknowledges a mux was created.
+	OpAckMux
+
+	// OpRequest is a single request + response.
+	// MuxID is returned in response.
+	OpRequest
+
+	// OpResponse is a response to a single request.
+	// FlagPayloadIsErr is used to signify that the payload is a string error converted to byte slice.
+	// When a response is received, the mux is already removed from the remote.
+	OpResponse
+
+	// OpDisconnect instructs that remote wants to disconnect
+	OpDisconnect
+
+	// OpMerged is several operations merged into one.
+	OpMerged
+)
+
+const (
+	// FlagCRCxxh3 indicates that, the lower 32 bits of xxhash3 of the serialized
+	// message will be sent after the serialized message as little endian.
+	FlagCRCxxh3 Flags = 1 << iota
+
+	// FlagEOF the stream (either direction) is at EOF.
+	FlagEOF
+
+	// FlagStateless indicates the message is stateless.
+	// This will retain clients across reconnections or
+	// if sequence numbers are unexpected.
+	FlagStateless
+
+	// FlagPayloadIsErr can be used by individual ops to signify that
+	// The payload is a string error converted to byte slice.
+	FlagPayloadIsErr
+
+	// FlagPayloadIsZero means that payload is 0-length slice and not nil.
+	FlagPayloadIsZero
+
+	// FlagSubroute indicates that the message has subroute.
+	// Subroute will be 32 bytes long and added before any CRC.
+	FlagSubroute
+)
+
+// This struct cannot be changed and retain backwards compatibility.
+// If changed, endpoint version must be bumped.
+//
+//msgp:tuple message
+type message struct {
+	MuxID      uint64    // Mux to receive message if any.
+	Seq        uint32    // Sequence number.
+	DeadlineMS uint32    // If non-zero, milliseconds until deadline (max 1193h2m47.295s, ~49 days)
+	Handler    HandlerID // ID of handler if invoking a remote handler.
+	Op         Op        // Operation. Other fields change based on this value.
+	Flags      Flags     // Optional flags.
+	Payload    []byte    // Optional payload.
+}
+
+// Flags is a set of flags set on a message.
+type Flags uint8
+
+func (m message) String() string {
+	var res []string
+	if m.MuxID != 0 {
+		res = append(res, fmt.Sprintf("MuxID: %v", m.MuxID))
+	}
+	if m.Seq != 0 {
+		res = append(res, fmt.Sprintf("Seq: %v", m.Seq))
+	}
+	if m.DeadlineMS != 0 {
+		res = append(res, fmt.Sprintf("Deadline: %vms", m.DeadlineMS))
+	}
+	if m.Handler != handlerInvalid {
+		res = append(res, fmt.Sprintf("Handler: %v", m.Handler))
+	}
+	if m.Op != 0 {
+		res = append(res, fmt.Sprintf("Op: %v", m.Op))
+	}
+	res = append(res, fmt.Sprintf("Flags: %s", m.Flags.String()))
+	if len(m.Payload) != 0 {
+		res = append(res, fmt.Sprintf("Payload: %v", bytesOrLength(m.Payload)))
+	}
+	return "{" + strings.Join(res, ", ") + "}"
+}
+
+func (f Flags) String() string {
+	var res []string
+	if f&FlagCRCxxh3 != 0 {
+		res = append(res, "CRC")
+	}
+	if f&FlagEOF != 0 {
+		res = append(res, "EOF")
+	}
+	if f&FlagStateless != 0 {
+		res = append(res, "SL")
+	}
+	if f&FlagPayloadIsErr != 0 {
+		res = append(res, "ERR")
+	}
+	if f&FlagPayloadIsZero != 0 {
+		res = append(res, "ZERO")
+	}
+	if f&FlagSubroute != 0 {
+		res = append(res, "SUB")
+	}
+	return "[" + strings.Join(res, ",") + "]"
+}
+
+// Set one or more flags on f.
+func (f *Flags) Set(flags Flags) {
+	*f |= flags
+}
+
+// Clear one or more flags on f.
+func (f *Flags) Clear(flags Flags) {
+	*f &^= flags
+}
+
+// parse an incoming message.
+func (m *message) parse(b []byte) (*subHandlerID, []byte, error) {
+	var sub *subHandlerID
+	if m.Payload == nil {
+		m.Payload = GetByteBuffer()[:0]
+	}
+	h, err := m.UnmarshalMsg(b)
+	if err != nil {
+		return nil, nil, fmt.Errorf("read write: %v", err)
+	}
+	if len(m.Payload) == 0 && m.Flags&FlagPayloadIsZero == 0 {
+		PutByteBuffer(m.Payload)
+		m.Payload = nil
+	}
+	if m.Flags&FlagCRCxxh3 != 0 {
+		const hashLen = 4
+		if len(h) < hashLen {
+			return nil, nil, fmt.Errorf("want crc len 4, got %v", len(h))
+		}
+		got := uint32(xxh3.Hash(b[:len(b)-hashLen]))
+		want := binary.LittleEndian.Uint32(h[len(h)-hashLen:])
+		if got != want {
+			return nil, nil, fmt.Errorf("crc mismatch: 0x%08x (given) != 0x%08x (bytes)", want, got)
+		}
+		h = h[:len(h)-hashLen]
+	}
+	// Extract subroute if any.
+	if m.Flags&FlagSubroute != 0 {
+		if len(h) < 32 {
+			return nil, nil, fmt.Errorf("want subroute len 32, got %v", len(h))
+		}
+		subID := (*[32]byte)(h[len(h)-32:])
+		sub = (*subHandlerID)(subID)
+		// Add if more modifications to h is needed
+		h = h[:len(h)-32]
+	}
+	return sub, h, nil
+}
+
+// setZeroPayloadFlag will clear or set the FlagPayloadIsZero if
+// m.Payload is length 0, but not nil.
+func (m *message) setZeroPayloadFlag() {
+	m.Flags &^= FlagPayloadIsZero
+	if len(m.Payload) == 0 && m.Payload != nil {
+		m.Flags |= FlagPayloadIsZero
+	}
+}
+
+type receiver interface {
+	msgp.Unmarshaler
+	Op() Op
+}
+
+type sender interface {
+	msgp.MarshalSizer
+	Op() Op
+}
+
+type connectReq struct {
+	ID    [16]byte
+	Host  string
+	Time  time.Time
+	Token string
+}
+
+// addToken will add the token to the connect request.
+func (c *connectReq) addToken(fn AuthFn) {
+	c.Token = fn()
+}
+
+func (connectReq) Op() Op {
+	return OpConnect
+}
+
+type connectResp struct {
+	ID             [16]byte
+	Accepted       bool
+	RejectedReason string
+}
+
+func (connectResp) Op() Op {
+	return OpConnectResponse
+}
+
+type muxConnectError struct {
+	Error string
+}
+
+func (muxConnectError) Op() Op {
+	return OpMuxConnectError
+}
+
+type pongMsg struct {
+	NotFound bool      `msg:"nf"`
+	Err      *string   `msg:"e,allownil"`
+	T        time.Time `msg:"t"`
+}
+
+func (pongMsg) Op() Op {
+	return OpPong
+}
+
+type pingMsg struct {
+	T time.Time `msg:"t"`
+}
+
+func (pingMsg) Op() Op {
+	return OpPing
+}

internal/grid/msg_gen_test.go

@@ -0,0 +1,688 @@
+// Code generated by github.com/tinylib/msgp DO NOT EDIT.
+
+package grid
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/tinylib/msgp/msgp"
+)
+
+func TestMarshalUnmarshalconnectReq(t *testing.T) {
+	v := connectReq{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgconnectReq(b *testing.B) {
+	v := connectReq{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgconnectReq(b *testing.B) {
+	v := connectReq{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalconnectReq(b *testing.B) {
+	v := connectReq{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeconnectReq(t *testing.T) {
+	v := connectReq{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeconnectReq Msgsize() is inaccurate")
+	}
+
+	vn := connectReq{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeconnectReq(b *testing.B) {
+	v := connectReq{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeconnectReq(b *testing.B) {
+	v := connectReq{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalconnectResp(t *testing.T) {
+	v := connectResp{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgconnectResp(b *testing.B) {
+	v := connectResp{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgconnectResp(b *testing.B) {
+	v := connectResp{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalconnectResp(b *testing.B) {
+	v := connectResp{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeconnectResp(t *testing.T) {
+	v := connectResp{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeconnectResp Msgsize() is inaccurate")
+	}
+
+	vn := connectResp{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeconnectResp(b *testing.B) {
+	v := connectResp{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeconnectResp(b *testing.B) {
+	v := connectResp{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalmessage(t *testing.T) {
+	v := message{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgmessage(b *testing.B) {
+	v := message{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgmessage(b *testing.B) {
+	v := message{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalmessage(b *testing.B) {
+	v := message{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodemessage(t *testing.T) {
+	v := message{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodemessage Msgsize() is inaccurate")
+	}
+
+	vn := message{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodemessage(b *testing.B) {
+	v := message{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodemessage(b *testing.B) {
+	v := message{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalmuxConnectError(t *testing.T) {
+	v := muxConnectError{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgmuxConnectError(b *testing.B) {
+	v := muxConnectError{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgmuxConnectError(b *testing.B) {
+	v := muxConnectError{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalmuxConnectError(b *testing.B) {
+	v := muxConnectError{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodemuxConnectError(t *testing.T) {
+	v := muxConnectError{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodemuxConnectError Msgsize() is inaccurate")
+	}
+
+	vn := muxConnectError{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodemuxConnectError(b *testing.B) {
+	v := muxConnectError{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodemuxConnectError(b *testing.B) {
+	v := muxConnectError{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalpingMsg(t *testing.T) {
+	v := pingMsg{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgpingMsg(b *testing.B) {
+	v := pingMsg{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgpingMsg(b *testing.B) {
+	v := pingMsg{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalpingMsg(b *testing.B) {
+	v := pingMsg{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodepingMsg(t *testing.T) {
+	v := pingMsg{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodepingMsg Msgsize() is inaccurate")
+	}
+
+	vn := pingMsg{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodepingMsg(b *testing.B) {
+	v := pingMsg{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodepingMsg(b *testing.B) {
+	v := pingMsg{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalpongMsg(t *testing.T) {
+	v := pongMsg{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgpongMsg(b *testing.B) {
+	v := pongMsg{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgpongMsg(b *testing.B) {
+	v := pongMsg{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalpongMsg(b *testing.B) {
+	v := pongMsg{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodepongMsg(t *testing.T) {
+	v := pongMsg{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodepongMsg Msgsize() is inaccurate")
+	}
+
+	vn := pongMsg{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodepongMsg(b *testing.B) {
+	v := pongMsg{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodepongMsg(b *testing.B) {
+	v := pongMsg{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}

internal/grid/msg_string.go

@@ -0,0 +1,40 @@
+// Code generated by "stringer -type=Op -output=msg_string.go -trimprefix=Op msg.go"; DO NOT EDIT.
+
+package grid
+
+import "strconv"
+
+func _() {
+	// An "invalid array index" compiler error signifies that the constant values have changed.
+	// Re-run the stringer command to generate them again.
+	var x [1]struct{}
+	_ = x[OpConnect-1]
+	_ = x[OpConnectResponse-2]
+	_ = x[OpPing-3]
+	_ = x[OpPong-4]
+	_ = x[OpConnectMux-5]
+	_ = x[OpMuxConnectError-6]
+	_ = x[OpDisconnectClientMux-7]
+	_ = x[OpDisconnectServerMux-8]
+	_ = x[OpMuxClientMsg-9]
+	_ = x[OpMuxServerMsg-10]
+	_ = x[OpUnblockSrvMux-11]
+	_ = x[OpUnblockClMux-12]
+	_ = x[OpAckMux-13]
+	_ = x[OpRequest-14]
+	_ = x[OpResponse-15]
+	_ = x[OpDisconnect-16]
+	_ = x[OpMerged-17]
+}
+
+const _Op_name = "ConnectConnectResponsePingPongConnectMuxMuxConnectErrorDisconnectClientMuxDisconnectServerMuxMuxClientMsgMuxServerMsgUnblockSrvMuxUnblockClMuxAckMuxRequestResponseDisconnectMerged"
+
+var _Op_index = [...]uint8{0, 7, 22, 26, 30, 40, 55, 74, 93, 105, 117, 130, 142, 148, 155, 163, 173, 179}
+
+func (i Op) String() string {
+	i -= 1
+	if i >= Op(len(_Op_index)-1) {
+		return "Op(" + strconv.FormatInt(int64(i+1), 10) + ")"
+	}
+	return _Op_name[_Op_index[i]:_Op_index[i+1]]
+}

internal/grid/muxclient.go

@@ -0,0 +1,662 @@
+// Copyright (c) 2015-2023 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package grid
+
+import (
+	"context"
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	xioutil "github.com/minio/minio/internal/ioutil"
+	"github.com/zeebo/xxh3"
+)
+
+// muxClient is a stateful connection to a remote.
+type muxClient struct {
+	MuxID              uint64
+	SendSeq, RecvSeq   uint32
+	LastPong           int64
+	BaseFlags          Flags
+	ctx                context.Context
+	cancelFn           context.CancelCauseFunc
+	parent             *Connection
+	respWait           chan<- Response
+	respMu             sync.Mutex
+	singleResp         bool
+	closed             bool
+	stateless          bool
+	acked              bool
+	init               bool
+	deadline           time.Duration
+	outBlock           chan struct{}
+	subroute           *subHandlerID
+	respErr            atomic.Pointer[error]
+	clientPingInterval time.Duration
+}
+
+// Response is a response from the server.
+type Response struct {
+	Msg []byte
+	Err error
+}
+
+func newMuxClient(ctx context.Context, muxID uint64, parent *Connection) *muxClient {
+	ctx, cancelFn := context.WithCancelCause(ctx)
+	return &muxClient{
+		MuxID:              muxID,
+		ctx:                ctx,
+		cancelFn:           cancelFn,
+		parent:             parent,
+		LastPong:           time.Now().UnixNano(),
+		BaseFlags:          parent.baseFlags,
+		clientPingInterval: parent.clientPingInterval,
+	}
+}
+
+// roundtrip performs a roundtrip, returning the first response.
+// This cannot be used concurrently.
+func (m *muxClient) roundtrip(h HandlerID, req []byte) ([]byte, error) {
+	if m.init {
+		return nil, errors.New("mux client already used")
+	}
+	m.init = true
+	m.singleResp = true
+	msg := message{
+		Op:         OpRequest,
+		MuxID:      m.MuxID,
+		Handler:    h,
+		Flags:      m.BaseFlags | FlagEOF,
+		Payload:    req,
+		DeadlineMS: uint32(m.deadline.Milliseconds()),
+	}
+	if m.subroute != nil {
+		msg.Flags |= FlagSubroute
+	}
+	ch := make(chan Response, 1)
+	m.respMu.Lock()
+	if m.closed {
+		m.respMu.Unlock()
+		return nil, ErrDisconnected
+	}
+	m.respWait = ch
+	m.respMu.Unlock()
+	ctx := m.ctx
+
+	// Add deadline if none.
+	if msg.DeadlineMS == 0 {
+		msg.DeadlineMS = uint32(defaultSingleRequestTimeout / time.Millisecond)
+		var cancel context.CancelFunc
+		ctx, cancel = context.WithTimeout(ctx, defaultSingleRequestTimeout)
+		defer cancel()
+	}
+	// Send request
+	if err := m.send(msg); err != nil {
+		return nil, err
+	}
+	if debugReqs {
+		fmt.Println(m.MuxID, m.parent.String(), "SEND")
+	}
+	// Wait for response or context.
+	select {
+	case v, ok := <-ch:
+		if !ok {
+			return nil, ErrDisconnected
+		}
+		if debugReqs && v.Err != nil {
+			v.Err = fmt.Errorf("%d %s RESP ERR: %w", m.MuxID, m.parent.String(), v.Err)
+		}
+		return v.Msg, v.Err
+	case <-ctx.Done():
+		if debugReqs {
+			return nil, fmt.Errorf("%d %s ERR: %w", m.MuxID, m.parent.String(), context.Cause(ctx))
+		}
+		return nil, context.Cause(ctx)
+	}
+}
+
+// send the message. msg.Seq and msg.MuxID will be set
+func (m *muxClient) send(msg message) error {
+	m.respMu.Lock()
+	defer m.respMu.Unlock()
+	if m.closed {
+		return errors.New("mux client closed")
+	}
+	return m.sendLocked(msg)
+}
+
+// sendLocked the message. msg.Seq and msg.MuxID will be set.
+// m.respMu must be held.
+func (m *muxClient) sendLocked(msg message) error {
+	dst := GetByteBufferCap(msg.Msgsize())
+	msg.Seq = m.SendSeq
+	msg.MuxID = m.MuxID
+	msg.Flags |= m.BaseFlags
+	if debugPrint {
+		fmt.Println("Client sending", &msg, "to", m.parent.Remote)
+	}
+	m.SendSeq++
+
+	dst, err := msg.MarshalMsg(dst)
+	if err != nil {
+		return err
+	}
+	if msg.Flags&FlagSubroute != 0 {
+		if m.subroute == nil {
+			return fmt.Errorf("internal error: subroute not defined on client")
+		}
+		hid := m.subroute.withHandler(msg.Handler)
+		before := len(dst)
+		dst = append(dst, hid[:]...)
+		if debugPrint {
+			fmt.Println("Added subroute", hid.String(), "to message", msg, "len", len(dst)-before)
+		}
+	}
+	if msg.Flags&FlagCRCxxh3 != 0 {
+		h := xxh3.Hash(dst)
+		dst = binary.LittleEndian.AppendUint32(dst, uint32(h))
+	}
+	return m.parent.send(m.ctx, dst)
+}
+
+// RequestStateless will send a single payload request and stream back results.
+// req may not be read/written to after calling.
+// TODO: Not implemented
+func (m *muxClient) RequestStateless(h HandlerID, req []byte, out chan<- Response) {
+	if m.init {
+		out <- Response{Err: errors.New("mux client already used")}
+	}
+	m.init = true
+
+	// Try to grab an initial block.
+	m.singleResp = false
+	msg := message{
+		Op:         OpConnectMux,
+		Handler:    h,
+		Flags:      FlagEOF,
+		Payload:    req,
+		DeadlineMS: uint32(m.deadline.Milliseconds()),
+	}
+	msg.setZeroPayloadFlag()
+	if m.subroute != nil {
+		msg.Flags |= FlagSubroute
+	}
+
+	// Send...
+	err := m.send(msg)
+	if err != nil {
+		out <- Response{Err: err}
+		return
+	}
+
+	// Route directly to output.
+	m.respWait = out
+}
+
+// RequestStream will send a single payload request and stream back results.
+// 'requests' can be nil, in which case only req is sent as input.
+// It will however take less resources.
+func (m *muxClient) RequestStream(h HandlerID, payload []byte, requests chan []byte, responses chan Response) (*Stream, error) {
+	if m.init {
+		return nil, errors.New("mux client already used")
+	}
+	if responses == nil {
+		return nil, errors.New("RequestStream: responses channel is nil")
+	}
+	m.init = true
+	m.respMu.Lock()
+	if m.closed {
+		m.respMu.Unlock()
+		return nil, ErrDisconnected
+	}
+	m.respWait = responses // Route directly to output.
+	m.respMu.Unlock()
+
+	// Try to grab an initial block.
+	m.singleResp = false
+	m.RecvSeq = m.SendSeq // Sync
+	if cap(requests) > 0 {
+		m.outBlock = make(chan struct{}, cap(requests))
+	}
+	msg := message{
+		Op:         OpConnectMux,
+		Handler:    h,
+		Payload:    payload,
+		DeadlineMS: uint32(m.deadline.Milliseconds()),
+	}
+	msg.setZeroPayloadFlag()
+	if requests == nil {
+		msg.Flags |= FlagEOF
+	}
+	if m.subroute != nil {
+		msg.Flags |= FlagSubroute
+	}
+
+	// Send...
+	err := m.send(msg)
+	if err != nil {
+		return nil, err
+	}
+	if debugPrint {
+		fmt.Println("Connecting Mux", m.MuxID, ",to", m.parent.Remote)
+	}
+
+	// Space for one message and an error.
+	responseCh := make(chan Response, 1)
+
+	// Spawn simple disconnect
+	if requests == nil {
+		go m.handleOneWayStream(responseCh, responses)
+		return &Stream{responses: responseCh, Requests: nil, ctx: m.ctx, cancel: m.cancelFn, muxID: m.MuxID}, nil
+	}
+
+	// Deliver responses and send unblocks back to the server.
+	go m.handleTwowayResponses(responseCh, responses)
+	go m.handleTwowayRequests(responses, requests)
+
+	return &Stream{responses: responseCh, Requests: requests, ctx: m.ctx, cancel: m.cancelFn, muxID: m.MuxID}, nil
+}
+
+func (m *muxClient) addErrorNonBlockingClose(respHandler chan<- Response, err error) {
+	m.respMu.Lock()
+	defer m.respMu.Unlock()
+	if !m.closed {
+		m.respErr.Store(&err)
+		// Do not block.
+		select {
+		case respHandler <- Response{Err: err}:
+			xioutil.SafeClose(respHandler)
+		default:
+			go func() {
+				respHandler <- Response{Err: err}
+				xioutil.SafeClose(respHandler)
+			}()
+		}
+		gridLogIf(m.ctx, m.sendLocked(message{Op: OpDisconnectServerMux, MuxID: m.MuxID}))
+		m.closed = true
+	}
+}
+
+// respHandler
+func (m *muxClient) handleOneWayStream(respHandler chan<- Response, respServer <-chan Response) {
+	if debugPrint {
+		start := time.Now()
+		defer func() {
+			fmt.Println("Mux", m.MuxID, "Request took", time.Since(start).Round(time.Millisecond))
+		}()
+	}
+	defer func() {
+		// addErrorNonBlockingClose will close the response channel
+		// - maybe async, so we shouldn't do it here.
+		if m.respErr.Load() == nil {
+			xioutil.SafeClose(respHandler)
+		}
+	}()
+	var pingTimer <-chan time.Time
+	if m.deadline == 0 || m.deadline > m.clientPingInterval {
+		ticker := time.NewTicker(m.clientPingInterval)
+		defer ticker.Stop()
+		pingTimer = ticker.C
+		atomic.StoreInt64(&m.LastPong, time.Now().UnixNano())
+	}
+	defer m.parent.deleteMux(false, m.MuxID)
+	for {
+		select {
+		case <-m.ctx.Done():
+			if debugPrint {
+				fmt.Println("Client sending disconnect to mux", m.MuxID)
+			}
+			err := context.Cause(m.ctx)
+			if !errors.Is(err, errStreamEOF) {
+				m.addErrorNonBlockingClose(respHandler, err)
+			}
+			return
+		case resp, ok := <-respServer:
+			if !ok {
+				return
+			}
+		sendResp:
+			select {
+			case respHandler <- resp:
+				m.respMu.Lock()
+				if !m.closed {
+					gridLogIf(m.ctx, m.sendLocked(message{Op: OpUnblockSrvMux, MuxID: m.MuxID}))
+				}
+				m.respMu.Unlock()
+			case <-m.ctx.Done():
+				// Client canceled. Don't block.
+				// Next loop will catch it.
+			case <-pingTimer:
+				if !m.doPing(respHandler) {
+					return
+				}
+				goto sendResp
+			}
+		case <-pingTimer:
+			if !m.doPing(respHandler) {
+				return
+			}
+		}
+	}
+}
+
+// doPing checks last ping time and sends another ping.
+func (m *muxClient) doPing(respHandler chan<- Response) (ok bool) {
+	m.respMu.Lock()
+	if m.closed {
+		m.respMu.Unlock()
+		// Already closed. This is not an error state;
+		// we may just be delivering the last responses.
+		return true
+	}
+
+	// Only check ping when not closed.
+	if got := time.Since(time.Unix(0, atomic.LoadInt64(&m.LastPong))); got > m.clientPingInterval*2 {
+		m.respMu.Unlock()
+		if debugPrint {
+			fmt.Printf("Mux %d: last pong %v ago, disconnecting\n", m.MuxID, got)
+		}
+		m.addErrorNonBlockingClose(respHandler, ErrDisconnected)
+		return false
+	}
+
+	// Send new ping
+	err := m.sendLocked(message{Op: OpPing, MuxID: m.MuxID})
+	m.respMu.Unlock()
+	if err != nil {
+		m.addErrorNonBlockingClose(respHandler, err)
+	}
+	return err == nil
+}
+
+// responseCh is the channel to that goes to the requester.
+// internalResp is the channel that comes from the server.
+func (m *muxClient) handleTwowayResponses(responseCh chan<- Response, internalResp <-chan Response) {
+	defer func() {
+		m.parent.deleteMux(false, m.MuxID)
+		// addErrorNonBlockingClose will close the response channel.
+		xioutil.SafeClose(responseCh)
+	}()
+
+	// Cancelation and errors are handled by handleTwowayRequests below.
+	for resp := range internalResp {
+		m.send(message{Op: OpUnblockSrvMux, MuxID: m.MuxID})
+		responseCh <- resp
+	}
+}
+
+func (m *muxClient) handleTwowayRequests(errResp chan<- Response, requests <-chan []byte) {
+	var errState bool
+	if debugPrint {
+		start := time.Now()
+		defer func() {
+			fmt.Println("Mux", m.MuxID, "Request took", time.Since(start).Round(time.Millisecond))
+		}()
+	}
+
+	var pingTimer <-chan time.Time
+	if m.deadline == 0 || m.deadline > m.clientPingInterval {
+		ticker := time.NewTicker(m.clientPingInterval)
+		defer ticker.Stop()
+		pingTimer = ticker.C
+		atomic.StoreInt64(&m.LastPong, time.Now().UnixNano())
+	}
+
+	// Listen for client messages.
+reqLoop:
+	for !errState {
+		select {
+		case <-m.ctx.Done():
+			if debugPrint {
+				fmt.Println("Client sending disconnect to mux", m.MuxID)
+			}
+			m.addErrorNonBlockingClose(errResp, context.Cause(m.ctx))
+			errState = true
+			continue
+		case <-pingTimer:
+			if !m.doPing(errResp) {
+				errState = true
+				continue
+			}
+		case req, ok := <-requests:
+			if !ok {
+				// Done send EOF
+				if debugPrint {
+					fmt.Println("Client done, sending EOF to mux", m.MuxID)
+				}
+				msg := message{
+					Op:    OpMuxClientMsg,
+					MuxID: m.MuxID,
+					Flags: FlagEOF,
+				}
+				msg.setZeroPayloadFlag()
+				err := m.send(msg)
+				if err != nil {
+					m.addErrorNonBlockingClose(errResp, err)
+				}
+				break reqLoop
+			}
+			// Grab a send token.
+		sendReq:
+			select {
+			case <-m.ctx.Done():
+				m.addErrorNonBlockingClose(errResp, context.Cause(m.ctx))
+				errState = true
+				continue
+			case <-pingTimer:
+				if !m.doPing(errResp) {
+					errState = true
+					continue
+				}
+				goto sendReq
+			case <-m.outBlock:
+			}
+			msg := message{
+				Op:      OpMuxClientMsg,
+				MuxID:   m.MuxID,
+				Seq:     1,
+				Payload: req,
+			}
+			msg.setZeroPayloadFlag()
+			err := m.send(msg)
+			PutByteBuffer(req)
+			if err != nil {
+				m.addErrorNonBlockingClose(errResp, err)
+				errState = true
+				continue
+			}
+			msg.Seq++
+		}
+	}
+
+	if errState {
+		// Drain requests.
+		for {
+			select {
+			case r, ok := <-requests:
+				if !ok {
+					return
+				}
+				PutByteBuffer(r)
+			default:
+				return
+			}
+		}
+	}
+
+	for !errState {
+		select {
+		case <-m.ctx.Done():
+			if debugPrint {
+				fmt.Println("Client sending disconnect to mux", m.MuxID)
+			}
+			m.addErrorNonBlockingClose(errResp, context.Cause(m.ctx))
+			return
+		case <-pingTimer:
+			errState = !m.doPing(errResp)
+		}
+	}
+}
+
+// checkSeq will check if sequence number is correct and increment it by 1.
+func (m *muxClient) checkSeq(seq uint32) (ok bool) {
+	if seq != m.RecvSeq {
+		if debugPrint {
+			fmt.Printf("MuxID: %d client, expected sequence %d, got %d\n", m.MuxID, m.RecvSeq, seq)
+		}
+		m.addResponse(Response{Err: ErrIncorrectSequence})
+		return false
+	}
+	m.RecvSeq++
+	return true
+}
+
+// response will send handleIncoming response to client.
+// may never block.
+// Should return whether the next call would block.
+func (m *muxClient) response(seq uint32, r Response) {
+	if debugReqs {
+		fmt.Println(m.MuxID, m.parent.String(), "RESP")
+	}
+	if debugPrint {
+		fmt.Printf("mux %d: got msg seqid %d, payload length: %d, err:%v\n", m.MuxID, seq, len(r.Msg), r.Err)
+	}
+	if !m.checkSeq(seq) {
+		if debugReqs {
+			fmt.Println(m.MuxID, m.parent.String(), "CHECKSEQ FAIL", m.RecvSeq, seq)
+		}
+		PutByteBuffer(r.Msg)
+		r.Msg = nil
+		r.Err = ErrIncorrectSequence
+		m.addResponse(r)
+		return
+	}
+	atomic.StoreInt64(&m.LastPong, time.Now().UnixNano())
+	ok := m.addResponse(r)
+	if !ok {
+		PutByteBuffer(r.Msg)
+	}
+}
+
+var errStreamEOF = errors.New("stream EOF")
+
+// error is a message from the server to disconnect.
+func (m *muxClient) error(err RemoteErr) {
+	if debugPrint {
+		fmt.Printf("mux %d: got remote err:%v\n", m.MuxID, string(err))
+	}
+	m.addResponse(Response{Err: &err})
+}
+
+func (m *muxClient) ack(seq uint32) {
+	if !m.checkSeq(seq) {
+		return
+	}
+	if m.acked || m.outBlock == nil {
+		return
+	}
+	available := cap(m.outBlock)
+	for range available {
+		m.outBlock <- struct{}{}
+	}
+	m.acked = true
+}
+
+func (m *muxClient) unblockSend(seq uint32) {
+	if !m.checkSeq(seq) {
+		return
+	}
+	select {
+	case m.outBlock <- struct{}{}:
+	default:
+		gridLogIf(m.ctx, errors.New("output unblocked overflow"))
+	}
+}
+
+func (m *muxClient) pong(msg pongMsg) {
+	if msg.NotFound || msg.Err != nil {
+		err := errors.New("remote terminated call")
+		if msg.Err != nil {
+			err = fmt.Errorf("remove pong failed: %v", &msg.Err)
+		}
+		m.addResponse(Response{Err: err})
+		return
+	}
+	atomic.StoreInt64(&m.LastPong, time.Now().UnixNano())
+}
+
+// addResponse will add a response to the response channel.
+// This function will never block
+func (m *muxClient) addResponse(r Response) (ok bool) {
+	m.respMu.Lock()
+	defer m.respMu.Unlock()
+	if m.closed {
+		return false
+	}
+	select {
+	case m.respWait <- r:
+		if r.Err != nil {
+			if debugPrint {
+				fmt.Println("Closing mux", m.MuxID, "due to error:", r.Err)
+			}
+			m.closeLocked()
+		}
+		return true
+	default:
+		if m.stateless {
+			// Drop message if not stateful.
+			return ok
+		}
+		err := errors.New("INTERNAL ERROR: Response was blocked")
+		gridLogIf(m.ctx, err)
+		m.closeLocked()
+		return false
+	}
+}
+
+func (m *muxClient) close() {
+	if debugPrint {
+		fmt.Println("closing outgoing mux", m.MuxID)
+	}
+	if !m.respMu.TryLock() {
+		// Cancel before locking - will unblock any pending sends.
+		if m.cancelFn != nil {
+			m.cancelFn(context.Canceled)
+		}
+		// Wait for senders to release.
+		m.respMu.Lock()
+	}
+
+	defer m.respMu.Unlock()
+	m.closeLocked()
+}
+
+func (m *muxClient) closeLocked() {
+	if m.closed {
+		return
+	}
+	// We hold the lock, so nobody can modify m.respWait while we're closing.
+	if m.respWait != nil {
+		xioutil.SafeClose(m.respWait)
+		m.respWait = nil
+	}
+	m.closed = true
+}

internal/grid/muxserver.go

@@ -0,0 +1,392 @@
+// Copyright (c) 2015-2023 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package grid
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	xioutil "github.com/minio/minio/internal/ioutil"
+)
+
+type muxServer struct {
+	ID                 uint64
+	LastPing           int64
+	SendSeq, RecvSeq   uint32
+	Resp               chan []byte
+	BaseFlags          Flags
+	ctx                context.Context
+	cancel             context.CancelFunc
+	inbound            chan []byte
+	parent             *Connection
+	sendMu             sync.Mutex
+	recvMu             sync.Mutex
+	outBlock           chan struct{}
+	clientPingInterval time.Duration
+}
+
+func newMuxStateless(ctx context.Context, msg message, c *Connection, handler StatelessHandler) *muxServer {
+	var cancel context.CancelFunc
+	ctx = setCaller(ctx, c.remote)
+	if msg.DeadlineMS > 0 {
+		ctx, cancel = context.WithTimeout(ctx, time.Duration(msg.DeadlineMS)*time.Millisecond)
+	} else {
+		ctx, cancel = context.WithCancel(ctx)
+	}
+	m := muxServer{
+		ID:        msg.MuxID,
+		RecvSeq:   msg.Seq + 1,
+		SendSeq:   msg.Seq,
+		ctx:       ctx,
+		cancel:    cancel,
+		parent:    c,
+		LastPing:  time.Now().Unix(),
+		BaseFlags: c.baseFlags,
+	}
+	go func() {
+		// TODO: Handle
+	}()
+
+	return &m
+}
+
+func newMuxStream(ctx context.Context, msg message, c *Connection, handler StreamHandler) *muxServer {
+	var cancel context.CancelFunc
+	ctx = setCaller(ctx, c.remote)
+	if len(handler.Subroute) > 0 {
+		ctx = setSubroute(ctx, handler.Subroute)
+	}
+	if msg.DeadlineMS > 0 {
+		ctx, cancel = context.WithTimeout(ctx, time.Duration(msg.DeadlineMS)*time.Millisecond+c.addDeadline)
+	} else {
+		ctx, cancel = context.WithCancel(ctx)
+	}
+
+	send := make(chan []byte)
+	inboundCap, outboundCap := handler.InCapacity, handler.OutCapacity
+	if outboundCap <= 0 {
+		outboundCap = 1
+	}
+
+	m := muxServer{
+		ID:                 msg.MuxID,
+		RecvSeq:            msg.Seq + 1,
+		SendSeq:            msg.Seq,
+		ctx:                ctx,
+		cancel:             cancel,
+		parent:             c,
+		inbound:            nil,
+		outBlock:           make(chan struct{}, outboundCap),
+		LastPing:           time.Now().Unix(),
+		BaseFlags:          c.baseFlags,
+		clientPingInterval: c.clientPingInterval,
+	}
+	// Acknowledge Mux created.
+	// Send async.
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		var ack message
+		ack.Op = OpAckMux
+		ack.Flags = m.BaseFlags
+		ack.MuxID = m.ID
+		m.send(ack)
+		if debugPrint {
+			fmt.Println("connected stream mux:", ack.MuxID)
+		}
+	}()
+
+	// Data inbound to the handler
+	var handlerIn chan []byte
+	if inboundCap > 0 {
+		m.inbound = make(chan []byte, inboundCap)
+		handlerIn = make(chan []byte, 1)
+		go func(inbound chan []byte) {
+			wg.Wait()
+			defer xioutil.SafeClose(handlerIn)
+			m.handleInbound(c, inbound, handlerIn)
+		}(m.inbound)
+	}
+	// Fill outbound block.
+	// Each token represents a message that can be sent to the client without blocking.
+	// The client will refill the tokens as they confirm delivery of the messages.
+	for range outboundCap {
+		m.outBlock <- struct{}{}
+	}
+
+	// Handler goroutine.
+	var handlerErr atomic.Pointer[RemoteErr]
+	go func() {
+		wg.Wait()
+		defer xioutil.SafeClose(send)
+		err := m.handleRequests(ctx, msg, send, handler, handlerIn)
+		if err != nil {
+			handlerErr.Store(err)
+		}
+	}()
+
+	// Response sender goroutine...
+	go func(outBlock <-chan struct{}) {
+		wg.Wait()
+		defer m.parent.deleteMux(true, m.ID)
+		m.sendResponses(ctx, send, c, &handlerErr, outBlock)
+	}(m.outBlock)
+
+	// Remote aliveness check if needed.
+	if msg.DeadlineMS == 0 || msg.DeadlineMS > uint32(4*c.clientPingInterval/time.Millisecond) {
+		go func() {
+			wg.Wait()
+			m.checkRemoteAlive()
+		}()
+	}
+	return &m
+}
+
+// handleInbound sends unblocks when we have delivered the message to the handler.
+func (m *muxServer) handleInbound(c *Connection, inbound <-chan []byte, handlerIn chan<- []byte) {
+	for {
+		select {
+		case <-m.ctx.Done():
+			return
+		case in, ok := <-inbound:
+			if !ok {
+				return
+			}
+			select {
+			case <-m.ctx.Done():
+				return
+			case handlerIn <- in:
+				m.send(message{Op: OpUnblockClMux, MuxID: m.ID, Flags: c.baseFlags})
+			}
+		}
+	}
+}
+
+// sendResponses will send responses to the client.
+func (m *muxServer) sendResponses(ctx context.Context, toSend <-chan []byte, c *Connection, handlerErr *atomic.Pointer[RemoteErr], outBlock <-chan struct{}) {
+	for {
+		// Process outgoing message.
+		var payload []byte
+		var ok bool
+		select {
+		case payload, ok = <-toSend:
+		case <-ctx.Done():
+			return
+		}
+		select {
+		case <-ctx.Done():
+			return
+		case <-outBlock:
+		}
+		msg := message{
+			MuxID: m.ID,
+			Op:    OpMuxServerMsg,
+			Flags: c.baseFlags,
+		}
+		if !ok {
+			hErr := handlerErr.Load()
+			if debugPrint {
+				fmt.Println("muxServer: Mux", m.ID, "send EOF", hErr)
+			}
+			msg.Flags |= FlagEOF
+			if hErr != nil {
+				msg.Flags |= FlagPayloadIsErr
+				msg.Payload = []byte(*hErr)
+			}
+			msg.setZeroPayloadFlag()
+			m.send(msg)
+			return
+		}
+		msg.Payload = payload
+		msg.setZeroPayloadFlag()
+		m.send(msg)
+	}
+}
+
+// handleRequests will handle the requests from the client and call the handler function.
+func (m *muxServer) handleRequests(ctx context.Context, msg message, send chan<- []byte, handler StreamHandler, handlerIn <-chan []byte) (handlerErr *RemoteErr) {
+	start := time.Now()
+	defer func() {
+		if debugPrint {
+			fmt.Println("Mux", m.ID, "Handler took", time.Since(start).Round(time.Millisecond))
+		}
+		if r := recover(); r != nil {
+			gridLogIf(ctx, fmt.Errorf("grid handler (%v) panic: %v", msg.Handler, r))
+			err := RemoteErr(fmt.Sprintf("handler panic: %v", r))
+			handlerErr = &err
+		}
+		if debugPrint {
+			fmt.Println("muxServer: Mux", m.ID, "Returned with", handlerErr)
+		}
+	}()
+	// handlerErr is guarded by 'send' channel.
+	handlerErr = handler.Handle(ctx, msg.Payload, handlerIn, send)
+	return handlerErr
+}
+
+// checkRemoteAlive will check if the remote is alive.
+func (m *muxServer) checkRemoteAlive() {
+	t := time.NewTicker(m.clientPingInterval)
+	defer t.Stop()
+	for {
+		select {
+		case <-m.ctx.Done():
+			return
+		case <-t.C:
+			last := time.Since(time.Unix(atomic.LoadInt64(&m.LastPing), 0))
+			if last > 4*m.clientPingInterval {
+				gridLogIf(m.ctx, fmt.Errorf("canceling remote connection %s not seen for %v", m.parent, last))
+				m.close()
+				return
+			}
+		}
+	}
+}
+
+// checkSeq will check if sequence number is correct and increment it by 1.
+func (m *muxServer) checkSeq(seq uint32) (ok bool) {
+	if seq != m.RecvSeq {
+		if debugPrint {
+			fmt.Printf("expected sequence %d, got %d\n", m.RecvSeq, seq)
+		}
+		m.disconnect(fmt.Sprintf("receive sequence number mismatch. want %d, got %d", m.RecvSeq, seq), false)
+		return false
+	}
+	m.RecvSeq++
+	return true
+}
+
+func (m *muxServer) message(msg message) {
+	if debugPrint {
+		fmt.Printf("muxServer: received message %d, length %d\n", msg.Seq, len(msg.Payload))
+	}
+	if !m.checkSeq(msg.Seq) {
+		return
+	}
+	m.recvMu.Lock()
+	defer m.recvMu.Unlock()
+	if cap(m.inbound) == 0 {
+		m.disconnect("did not expect inbound message", true)
+		return
+	}
+	// Note, on EOF no value can be sent.
+	if msg.Flags&FlagEOF != 0 {
+		if len(msg.Payload) > 0 {
+			gridLogIf(m.ctx, fmt.Errorf("muxServer: EOF message with payload"))
+		}
+		if m.inbound != nil {
+			xioutil.SafeClose(m.inbound)
+			m.inbound = nil
+		}
+		return
+	}
+
+	select {
+	case <-m.ctx.Done():
+	case m.inbound <- msg.Payload:
+		if debugPrint {
+			fmt.Printf("muxServer: Sent seq %d to handler\n", msg.Seq)
+		}
+	default:
+		m.disconnect("handler blocked", true)
+	}
+}
+
+func (m *muxServer) unblockSend(seq uint32) {
+	if !m.checkSeq(seq) {
+		return
+	}
+	m.recvMu.Lock()
+	defer m.recvMu.Unlock()
+	if m.outBlock == nil {
+		// Closed
+		return
+	}
+	select {
+	case m.outBlock <- struct{}{}:
+	default:
+		gridLogIf(m.ctx, errors.New("output unblocked overflow"))
+	}
+}
+
+func (m *muxServer) ping(seq uint32) pongMsg {
+	if !m.checkSeq(seq) {
+		msg := fmt.Sprintf("receive sequence number mismatch. want %d, got %d", m.RecvSeq, seq)
+		return pongMsg{Err: &msg}
+	}
+	select {
+	case <-m.ctx.Done():
+		err := context.Cause(m.ctx).Error()
+		return pongMsg{Err: &err}
+	default:
+		atomic.StoreInt64(&m.LastPing, time.Now().Unix())
+		return pongMsg{}
+	}
+}
+
+// disconnect will disconnect the mux.
+// m.recvMu must be locked when calling this function.
+func (m *muxServer) disconnect(msg string, locked bool) {
+	if debugPrint {
+		fmt.Println("Mux", m.ID, "disconnecting. Reason:", msg)
+	}
+	if msg != "" {
+		m.send(message{Op: OpMuxServerMsg, MuxID: m.ID, Flags: FlagPayloadIsErr | FlagEOF, Payload: []byte(msg)})
+	} else {
+		m.send(message{Op: OpDisconnectClientMux, MuxID: m.ID})
+	}
+	// Unlock, since we are calling deleteMux, which will call close - which will lock recvMu.
+	if locked {
+		m.recvMu.Unlock()
+		defer m.recvMu.Lock()
+	}
+	m.parent.deleteMux(true, m.ID)
+}
+
+func (m *muxServer) send(msg message) {
+	m.sendMu.Lock()
+	defer m.sendMu.Unlock()
+	msg.MuxID = m.ID
+	msg.Seq = m.SendSeq
+	m.SendSeq++
+	if debugPrint {
+		fmt.Printf("Mux %d, Sending %+v\n", m.ID, msg)
+	}
+	gridLogIf(m.ctx, m.parent.queueMsg(msg, nil))
+}
+
+func (m *muxServer) close() {
+	m.cancel()
+	m.recvMu.Lock()
+	defer m.recvMu.Unlock()
+
+	if m.inbound != nil {
+		xioutil.SafeClose(m.inbound)
+		m.inbound = nil
+	}
+
+	if m.outBlock != nil {
+		xioutil.SafeClose(m.outBlock)
+		m.outBlock = nil
+	}
+}

internal/grid/stream.go

@@ -0,0 +1,114 @@
+// Copyright (c) 2015-2023 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package grid
+
+import (
+	"context"
+	"errors"
+)
+
+// A Stream is a two-way stream.
+// All responses *must* be read by the caller.
+// If the call is canceled through the context,
+// the appropriate error will be returned.
+type Stream struct {
+	// responses from the remote server.
+	// Channel will be closed after error or when remote closes.
+	// All responses *must* be read by the caller until either an error is returned or the channel is closed.
+	// Canceling the context will cause the context cancellation error to be returned.
+	responses <-chan Response
+	cancel    context.CancelCauseFunc
+
+	// Requests sent to the server.
+	// If the handler is defined with 0 incoming capacity this will be nil.
+	// Channel *must* be closed to signal the end of the stream.
+	// If the request context is canceled, the stream will no longer process requests.
+	// Requests sent cannot be used any further by the called.
+	Requests chan<- []byte
+
+	muxID uint64
+	ctx   context.Context
+}
+
+// Send a payload to the remote server.
+func (s *Stream) Send(b []byte) error {
+	if s.Requests == nil {
+		return errors.New("stream does not accept requests")
+	}
+	select {
+	case s.Requests <- b:
+		return nil
+	case <-s.ctx.Done():
+		return context.Cause(s.ctx)
+	}
+}
+
+// Results returns the results from the remote server one by one.
+// If any error is returned by the callback, the stream will be canceled.
+// If the context is canceled, the stream will be canceled.
+func (s *Stream) Results(next func(b []byte) error) (err error) {
+	done := false
+	defer func() {
+		if s.cancel != nil {
+			s.cancel(err)
+		}
+
+		if !done {
+			// Drain channel.
+			for range s.responses {
+			}
+		}
+	}()
+	doneCh := s.ctx.Done()
+	for {
+		select {
+		case <-doneCh:
+			if err := context.Cause(s.ctx); !errors.Is(err, errStreamEOF) {
+				return err
+			}
+			// Fall through to be sure we have returned all responses.
+			doneCh = nil
+		case resp, ok := <-s.responses:
+			if !ok {
+				done = true
+				return nil
+			}
+			if resp.Err != nil {
+				s.cancel(resp.Err)
+				return resp.Err
+			}
+			err = next(resp.Msg)
+			if err != nil {
+				s.cancel(err)
+				return err
+			}
+		}
+	}
+}
+
+// Done will return a channel that will be closed when the stream is done.
+// This mirrors context.Done().
+func (s *Stream) Done() <-chan struct{} {
+	return s.ctx.Done()
+}
+
+// Err will return the error that caused the stream to end.
+// This mirrors context.Err().
+func (s *Stream) Err() error {
+	return s.ctx.Err()
+}

internal/grid/trace.go

@@ -0,0 +1,153 @@
+// Copyright (c) 2015-2023 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package grid
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/minio/madmin-go/v3"
+	"github.com/minio/minio/internal/pubsub"
+)
+
+// TraceParamsKey allows to pass trace parameters to the request via context.
+// This is only needed when un-typed requests are used.
+// MSS, map[string]string types are preferred, but any struct with exported fields will work.
+type TraceParamsKey struct{}
+
+// traceRequests adds request tracing to the connection.
+func (c *Connection) traceRequests(p *pubsub.PubSub[madmin.TraceInfo, madmin.TraceType]) {
+	c.trace = &tracer{
+		Publisher: p,
+		TraceType: madmin.TraceInternal,
+		Prefix:    "grid",
+		Local:     c.Local,
+		Remote:    c.Remote,
+		Subroute:  "",
+	}
+}
+
+// subroute adds a specific subroute to the request.
+func (c *tracer) subroute(subroute string) *tracer {
+	if c == nil {
+		return nil
+	}
+	c2 := *c
+	c2.Subroute = subroute
+	return &c2
+}
+
+type tracer struct {
+	Publisher *pubsub.PubSub[madmin.TraceInfo, madmin.TraceType]
+	TraceType madmin.TraceType
+	Prefix    string
+	Local     string
+	Remote    string
+	Subroute  string
+}
+
+const (
+	httpScheme  = "http://"
+	httpsScheme = "https://"
+)
+
+func (c *muxClient) traceRoundtrip(ctx context.Context, t *tracer, h HandlerID, req []byte) ([]byte, error) {
+	if t == nil || t.Publisher.NumSubscribers(t.TraceType) == 0 {
+		return c.roundtrip(h, req)
+	}
+
+	// Following trimming is needed for consistency between outputs with other internode traces.
+	local := strings.TrimPrefix(strings.TrimPrefix(t.Local, httpsScheme), httpScheme)
+	remote := strings.TrimPrefix(strings.TrimPrefix(t.Remote, httpsScheme), httpScheme)
+
+	start := time.Now()
+	body := bytesOrLength(req)
+	resp, err := c.roundtrip(h, req)
+	end := time.Now()
+	status := http.StatusOK
+	errString := ""
+	if err != nil {
+		errString = err.Error()
+		if IsRemoteErr(err) == nil {
+			status = http.StatusInternalServerError
+		} else {
+			status = http.StatusBadRequest
+		}
+	}
+
+	prefix := t.Prefix
+	if p := handlerPrefixes[h]; p != "" {
+		prefix = p
+	}
+	trace := madmin.TraceInfo{
+		TraceType: t.TraceType,
+		FuncName:  prefix + "." + h.String(),
+		NodeName:  remote,
+		Time:      start,
+		Duration:  end.Sub(start),
+		Path:      t.Subroute,
+		Error:     errString,
+		Bytes:     int64(len(req) + len(resp)),
+		HTTP: &madmin.TraceHTTPStats{
+			ReqInfo: madmin.TraceRequestInfo{
+				Time:    start,
+				Proto:   "grid",
+				Method:  "REQ",
+				Client:  local,
+				Headers: nil,
+				Path:    t.Subroute,
+				Body:    []byte(body),
+			},
+			RespInfo: madmin.TraceResponseInfo{
+				Time:       end,
+				Headers:    nil,
+				StatusCode: status,
+				Body:       []byte(bytesOrLength(resp)),
+			},
+			CallStats: madmin.TraceCallStats{
+				InputBytes:      len(req),
+				OutputBytes:     len(resp),
+				TimeToFirstByte: end.Sub(start),
+			},
+		},
+	}
+	// If the context contains a TraceParamsKey, add it to the trace path.
+	v := ctx.Value(TraceParamsKey{})
+	// Should match SingleHandler.Call checks.
+	switch typed := v.(type) {
+	case *MSS:
+		trace.Path += typed.ToQuery()
+	case map[string]string:
+		m := MSS(typed)
+		trace.Path += m.ToQuery()
+	case *URLValues:
+		trace.Path += typed.Values().Encode()
+	case *NoPayload, *Bytes:
+		trace.Path = fmt.Sprintf("%s?payload=%T", trace.Path, typed)
+	case string:
+		trace.Path = fmt.Sprintf("%s?%s", trace.Path, typed)
+	default:
+	}
+	trace.HTTP.ReqInfo.Path = trace.Path
+
+	t.Publisher.Publish(trace)
+	return resp, err
+}

internal/grid/types.go

@@ -0,0 +1,712 @@
+// Copyright (c) 2015-2023 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package grid
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"math"
+	"net/url"
+	"sort"
+	"strings"
+	"sync"
+
+	"github.com/minio/minio/internal/bpool"
+	"github.com/tinylib/msgp/msgp"
+)
+
+// Recycler will override the internal reuse in typed handlers.
+// When this is supported, the handler will not do internal pooling of objects,
+// call Recycle() when the object is no longer needed.
+// The recycler should handle nil pointers.
+type Recycler interface {
+	Recycle()
+}
+
+// MSS is a map[string]string that can be serialized.
+// It is not very efficient, but it is only used for easy parameter passing.
+type MSS map[string]string
+
+// Get returns the value for the given key.
+func (m *MSS) Get(key string) string {
+	if m == nil {
+		return ""
+	}
+	return (*m)[key]
+}
+
+// Set a key, value pair.
+func (m *MSS) Set(key, value string) {
+	if m == nil {
+		*m = mssPool.Get()
+	}
+	(*m)[key] = value
+}
+
+// UnmarshalMsg deserializes m from the provided byte slice and returns the
+// remainder of bytes.
+func (m *MSS) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	if m == nil {
+		return bts, errors.New("MSS: UnmarshalMsg on nil pointer")
+	}
+	if msgp.IsNil(bts) {
+		bts = bts[1:]
+		*m = nil
+		return bts, nil
+	}
+	var zb0002 uint32
+	zb0002, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err, "Values")
+		return o, err
+	}
+	dst := *m
+	if dst == nil {
+		dst = make(map[string]string, zb0002)
+	} else if len(dst) > 0 {
+		for key := range dst {
+			delete(dst, key)
+		}
+	}
+	for zb0002 > 0 {
+		var za0001 string
+		var za0002 string
+		zb0002--
+		za0001, bts, err = msgp.ReadStringBytes(bts)
+		if err != nil {
+			err = msgp.WrapError(err, "Values")
+			return o, err
+		}
+		za0002, bts, err = msgp.ReadStringBytes(bts)
+		if err != nil {
+			err = msgp.WrapError(err, "Values", za0001)
+			return o, err
+		}
+		dst[za0001] = za0002
+	}
+	*m = dst
+	return bts, nil
+}
+
+// MarshalMsg appends the bytes representation of b to the provided byte slice.
+func (m *MSS) MarshalMsg(bytes []byte) (o []byte, err error) {
+	if m == nil || *m == nil {
+		return msgp.AppendNil(bytes), nil
+	}
+	o = msgp.AppendMapHeader(bytes, uint32(len(*m)))
+	for za0001, za0002 := range *m {
+		o = msgp.AppendString(o, za0001)
+		o = msgp.AppendString(o, za0002)
+	}
+	return o, nil
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message.
+func (m *MSS) Msgsize() int {
+	if m == nil || *m == nil {
+		return msgp.NilSize
+	}
+	s := msgp.MapHeaderSize
+	for za0001, za0002 := range *m {
+		s += msgp.StringPrefixSize + len(za0001) + msgp.StringPrefixSize + len(za0002)
+	}
+	return s
+}
+
+// NewMSS returns a new MSS.
+func NewMSS() *MSS {
+	m := MSS(mssPool.Get())
+	clear(m)
+	return &m
+}
+
+// NewMSSWith returns a new MSS with the given map.
+func NewMSSWith(m map[string]string) *MSS {
+	m2 := MSS(m)
+	return &m2
+}
+
+var mssPool = bpool.Pool[map[string]string]{
+	New: func() map[string]string {
+		return make(map[string]string, 5)
+	},
+}
+
+// Recycle the underlying map.
+func (m *MSS) Recycle() {
+	if m != nil && *m != nil {
+		mssPool.Put(*m)
+		*m = nil
+	}
+}
+
+// ToQuery constructs a URL query string from the MSS, including "?" if there are any keys.
+func (m MSS) ToQuery() string {
+	if len(m) == 0 {
+		return ""
+	}
+	var buf strings.Builder
+	buf.WriteByte('?')
+	keys := make([]string, 0, len(m))
+	for k := range m {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	for _, k := range keys {
+		v := m[k]
+		keyEscaped := url.QueryEscape(k)
+		if buf.Len() > 1 {
+			buf.WriteByte('&')
+		}
+		buf.WriteString(keyEscaped)
+		buf.WriteByte('=')
+		buf.WriteString(url.QueryEscape(v))
+	}
+	return buf.String()
+}
+
+// NewBytes returns a new Bytes.
+// A slice is preallocated.
+func NewBytes() *Bytes {
+	b := Bytes(GetByteBuffer()[:0])
+	return &b
+}
+
+// NewBytesCap returns an empty Bytes with the given capacity.
+func NewBytesCap(size int) *Bytes {
+	b := Bytes(GetByteBufferCap(size))
+	return &b
+}
+
+// NewBytesWith returns a new Bytes with the provided content.
+// When sent as a parameter, the caller gives up ownership of the byte slice.
+// When returned as response, the handler also gives up ownership of the byte slice.
+func NewBytesWith(b []byte) *Bytes {
+	bb := Bytes(b)
+	return &bb
+}
+
+// NewBytesWithCopyOf returns a new byte slice with a copy of the provided content.
+func NewBytesWithCopyOf(b []byte) *Bytes {
+	if b == nil {
+		bb := Bytes(nil)
+		return &bb
+	}
+	bb := NewBytesCap(len(b))
+	*bb = append(*bb, b...)
+	return bb
+}
+
+// Bytes provides a byte slice that can be serialized.
+type Bytes []byte
+
+// UnmarshalMsg deserializes b from the provided byte slice and returns the
+// remainder of bytes.
+func (b *Bytes) UnmarshalMsg(bytes []byte) ([]byte, error) {
+	if b == nil {
+		return bytes, errors.New("Bytes: UnmarshalMsg on nil pointer")
+	}
+	if bytes, err := msgp.ReadNilBytes(bytes); err == nil {
+		if *b != nil {
+			PutByteBuffer(*b)
+		}
+		*b = nil
+		return bytes, nil
+	}
+	val, bytes, err := msgp.ReadBytesZC(bytes)
+	if err != nil {
+		return bytes, err
+	}
+	if cap(*b) >= len(val) {
+		*b = (*b)[:len(val)]
+		copy(*b, val)
+	} else {
+		if cap(*b) == 0 && len(val) <= maxBufferSize {
+			*b = GetByteBufferCap(len(val))
+		} else {
+			PutByteBuffer(*b)
+			*b = make([]byte, 0, len(val))
+		}
+		in := *b
+		in = append(in[:0], val...)
+		*b = in
+	}
+	return bytes, nil
+}
+
+// MarshalMsg appends the bytes representation of b to the provided byte slice.
+func (b *Bytes) MarshalMsg(bytes []byte) ([]byte, error) {
+	if b == nil || *b == nil {
+		return msgp.AppendNil(bytes), nil
+	}
+	return msgp.AppendBytes(bytes, *b), nil
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message.
+func (b *Bytes) Msgsize() int {
+	if b == nil || *b == nil {
+		return msgp.NilSize
+	}
+	return msgp.ArrayHeaderSize + len(*b)
+}
+
+// Recycle puts the Bytes back into the pool.
+func (b *Bytes) Recycle() {
+	if b != nil && *b != nil {
+		*b = (*b)[:0]
+		PutByteBuffer(*b)
+		*b = nil
+	}
+}
+
+// URLValues can be used for url.Values.
+type URLValues map[string][]string
+
+var urlValuesPool = bpool.Pool[map[string][]string]{
+	New: func() map[string][]string {
+		return make(map[string][]string, 10)
+	},
+}
+
+// NewURLValues returns a new URLValues.
+func NewURLValues() *URLValues {
+	u := URLValues(urlValuesPool.Get())
+	return &u
+}
+
+// NewURLValuesWith returns a new URLValues with the provided content.
+func NewURLValuesWith(values map[string][]string) *URLValues {
+	u := URLValues(values)
+	return &u
+}
+
+// Values returns the url.Values.
+// If u is nil, an empty url.Values is returned.
+// The values are a shallow copy of the underlying map.
+func (u *URLValues) Values() url.Values {
+	if u == nil {
+		return url.Values{}
+	}
+	return url.Values(*u)
+}
+
+// Recycle the underlying map.
+func (u *URLValues) Recycle() {
+	if *u != nil {
+		for key := range *u {
+			delete(*u, key)
+		}
+		val := map[string][]string(*u)
+		urlValuesPool.Put(val)
+		*u = nil
+	}
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (u URLValues) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, u.Msgsize())
+	o = msgp.AppendMapHeader(o, uint32(len(u)))
+	for zb0006, zb0007 := range u {
+		o = msgp.AppendString(o, zb0006)
+		o = msgp.AppendArrayHeader(o, uint32(len(zb0007)))
+		for zb0008 := range zb0007 {
+			o = msgp.AppendString(o, zb0007[zb0008])
+		}
+	}
+	return o, err
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (u *URLValues) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var zb0004 uint32
+	zb0004, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return o, err
+	}
+	if *u == nil {
+		*u = urlValuesPool.Get()
+	}
+	if len(*u) > 0 {
+		for key := range *u {
+			delete(*u, key)
+		}
+	}
+
+	for zb0004 > 0 {
+		var zb0001 string
+		var zb0002 []string
+		zb0004--
+		zb0001, bts, err = msgp.ReadStringBytes(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return o, err
+		}
+		var zb0005 uint32
+		zb0005, bts, err = msgp.ReadArrayHeaderBytes(bts)
+		if err != nil {
+			err = msgp.WrapError(err, zb0001)
+			return o, err
+		}
+		if cap(zb0002) >= int(zb0005) {
+			zb0002 = zb0002[:zb0005]
+		} else {
+			zb0002 = make([]string, zb0005)
+		}
+		for zb0003 := range zb0002 {
+			zb0002[zb0003], bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, zb0001, zb0003)
+				return o, err
+			}
+		}
+		(*u)[zb0001] = zb0002
+	}
+	o = bts
+	return o, err
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (u URLValues) Msgsize() (s int) {
+	s = msgp.MapHeaderSize
+	for zb0006, zb0007 := range u {
+		_ = zb0007
+		s += msgp.StringPrefixSize + len(zb0006) + msgp.ArrayHeaderSize
+		for zb0008 := range zb0007 {
+			s += msgp.StringPrefixSize + len(zb0007[zb0008])
+		}
+	}
+
+	return s
+}
+
+// JSONPool is a pool for JSON objects that unmarshal into T.
+type JSONPool[T any] struct {
+	pool    sync.Pool
+	emptySz int
+}
+
+// NewJSONPool returns a new JSONPool.
+func NewJSONPool[T any]() *JSONPool[T] {
+	var t T
+	sz := 128
+	if b, err := json.Marshal(t); err != nil {
+		sz = len(b)
+	}
+	return &JSONPool[T]{
+		pool: sync.Pool{
+			New: func() any {
+				var t T
+				return &t
+			},
+		},
+		emptySz: sz,
+	}
+}
+
+func (p *JSONPool[T]) new() *T {
+	var zero T
+	if t, ok := p.pool.Get().(*T); ok {
+		*t = zero
+		return t
+	}
+	return &zero
+}
+
+// JSON is a wrapper around a T object that can be serialized.
+// There is an internal value
+type JSON[T any] struct {
+	p   *JSONPool[T]
+	val *T
+}
+
+// NewJSON returns a new JSONPool.
+// No initial value is set.
+func (p *JSONPool[T]) NewJSON() *JSON[T] {
+	var j JSON[T]
+	j.p = p
+	return &j
+}
+
+// NewJSONWith returns a new JSON with the provided value.
+func (p *JSONPool[T]) NewJSONWith(val *T) *JSON[T] {
+	var j JSON[T]
+	j.p = p
+	j.val = val
+	return &j
+}
+
+// Value returns the underlying value.
+// If not set yet, a new value is created.
+func (j *JSON[T]) Value() *T {
+	if j.val == nil {
+		j.val = j.p.new()
+	}
+	return j.val
+}
+
+// ValueOrZero returns the underlying value.
+// If the underlying value is nil, a zero value is returned.
+func (j *JSON[T]) ValueOrZero() T {
+	if j == nil || j.val == nil {
+		var t T
+		return t
+	}
+	return *j.val
+}
+
+// Set the underlying value.
+func (j *JSON[T]) Set(v *T) {
+	j.val = v
+}
+
+// Recycle the underlying value.
+func (j *JSON[T]) Recycle() {
+	if j.val != nil {
+		j.p.pool.Put(j.val)
+		j.val = nil
+	}
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (j *JSON[T]) MarshalMsg(b []byte) (o []byte, err error) {
+	if j.val == nil {
+		return msgp.AppendNil(b), nil
+	}
+	buf := bytes.NewBuffer(GetByteBuffer()[:0])
+	defer func() {
+		PutByteBuffer(buf.Bytes())
+	}()
+	enc := json.NewEncoder(buf)
+	err = enc.Encode(j.val)
+	if err != nil {
+		return b, err
+	}
+	return msgp.AppendBytes(b, buf.Bytes()), nil
+}
+
+// UnmarshalMsg will JSON marshal the value and wrap as a msgp byte array.
+// Nil values are supported.
+func (j *JSON[T]) UnmarshalMsg(bytes []byte) ([]byte, error) {
+	if bytes, err := msgp.ReadNilBytes(bytes); err == nil {
+		if j.val != nil {
+			j.p.pool.Put(j.val)
+		}
+		j.val = nil
+		return bytes, nil
+	}
+	val, bytes, err := msgp.ReadBytesZC(bytes)
+	if err != nil {
+		return bytes, err
+	}
+	if j.val == nil {
+		j.val = j.p.new()
+	} else {
+		var t T
+		*j.val = t
+	}
+	return bytes, json.Unmarshal(val, j.val)
+}
+
+// Msgsize returns the size of an empty JSON object.
+func (j *JSON[T]) Msgsize() int {
+	return j.p.emptySz
+}
+
+// NoPayload is a type that can be used for handlers that do not use a payload.
+type NoPayload struct{}
+
+// Msgsize returns 0.
+func (p NoPayload) Msgsize() int {
+	return 0
+}
+
+// UnmarshalMsg satisfies the interface, but is a no-op.
+func (NoPayload) UnmarshalMsg(bytes []byte) ([]byte, error) {
+	return bytes, nil
+}
+
+// MarshalMsg satisfies the interface, but is a no-op.
+func (NoPayload) MarshalMsg(bytes []byte) ([]byte, error) {
+	return bytes, nil
+}
+
+// NewNoPayload returns an empty NoPayload struct.
+func NewNoPayload() NoPayload {
+	return NoPayload{}
+}
+
+// Recycle is a no-op.
+func (NoPayload) Recycle() {}
+
+// ArrayOf wraps an array of Messagepack compatible objects.
+type ArrayOf[T RoundTripper] struct {
+	aPool sync.Pool     // Arrays
+	ePool bpool.Pool[T] // Elements
+}
+
+// NewArrayOf returns a new ArrayOf.
+// You must provide a function that returns a new instance of T.
+func NewArrayOf[T RoundTripper](newFn func() T) *ArrayOf[T] {
+	return &ArrayOf[T]{
+		ePool: bpool.Pool[T]{New: func() T {
+			return newFn()
+		}},
+	}
+}
+
+// New returns a new empty Array.
+func (p *ArrayOf[T]) New() *Array[T] {
+	return &Array[T]{
+		p: p,
+	}
+}
+
+// NewWith returns a new Array with the provided value (not copied).
+func (p *ArrayOf[T]) NewWith(val []T) *Array[T] {
+	return &Array[T]{
+		p:   p,
+		val: val,
+	}
+}
+
+func (p *ArrayOf[T]) newA(sz uint32) []T {
+	t, ok := p.aPool.Get().(*[]T)
+	if !ok || t == nil {
+		return make([]T, 0, sz)
+	}
+	t2 := *t
+	return t2[:0]
+}
+
+func (p *ArrayOf[T]) putA(v []T) {
+	var zero T // nil
+	for i, t := range v {
+		//nolint:staticcheck // SA6002 IT IS A GENERIC VALUE!
+		p.ePool.Put(t)
+		v[i] = zero
+	}
+	if v != nil {
+		v = v[:0]
+		p.aPool.Put(&v)
+	}
+}
+
+func (p *ArrayOf[T]) newE() T {
+	return p.ePool.Get()
+}
+
+// Array provides a wrapper for an underlying array of serializable objects.
+type Array[T RoundTripper] struct {
+	p   *ArrayOf[T]
+	val []T
+}
+
+// Msgsize returns the size of the array in bytes.
+func (j *Array[T]) Msgsize() int {
+	if j.val == nil {
+		return msgp.NilSize
+	}
+	sz := msgp.ArrayHeaderSize
+	for _, v := range j.val {
+		sz += v.Msgsize()
+	}
+	return sz
+}
+
+// Value returns the underlying value.
+// Regular append mechanics should be observed.
+// If no value has been set yet, a new array is created.
+func (j *Array[T]) Value() []T {
+	if j.val == nil {
+		j.val = j.p.newA(10)
+	}
+	return j.val
+}
+
+// Append a value to the underlying array.
+// The returned Array is always the same as the one called.
+func (j *Array[T]) Append(v ...T) *Array[T] {
+	if j.val == nil {
+		j.val = j.p.newA(uint32(len(v)))
+	}
+	j.val = append(j.val, v...)
+	return j
+}
+
+// Set the underlying value.
+func (j *Array[T]) Set(val []T) {
+	j.val = val
+}
+
+// Recycle the underlying value.
+func (j *Array[T]) Recycle() {
+	if j.val != nil {
+		j.p.putA(j.val)
+		j.val = nil
+	}
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (j *Array[T]) MarshalMsg(b []byte) (o []byte, err error) {
+	if j.val == nil {
+		return msgp.AppendNil(b), nil
+	}
+	if uint64(len(j.val)) > math.MaxUint32 {
+		return b, errors.New("array: length of array exceeds math.MaxUint32")
+	}
+	b = msgp.AppendArrayHeader(b, uint32(len(j.val)))
+	for _, v := range j.val {
+		b, err = v.MarshalMsg(b)
+		if err != nil {
+			return b, err
+		}
+	}
+	return b, err
+}
+
+// UnmarshalMsg will JSON marshal the value and wrap as a msgp byte array.
+// Nil values are supported.
+func (j *Array[T]) UnmarshalMsg(bytes []byte) ([]byte, error) {
+	if bytes, err := msgp.ReadNilBytes(bytes); err == nil {
+		if j.val != nil {
+			j.p.putA(j.val)
+		}
+		j.val = nil
+		return bytes, nil
+	}
+	l, bytes, err := msgp.ReadArrayHeaderBytes(bytes)
+	if err != nil {
+		return bytes, err
+	}
+	if j.val == nil {
+		j.val = j.p.newA(l)
+	} else {
+		j.val = j.val[:0]
+	}
+	for range l {
+		v := j.p.newE()
+		bytes, err = v.UnmarshalMsg(bytes)
+		if err != nil {
+			return bytes, err
+		}
+		j.val = append(j.val, v)
+	}
+	return bytes, nil
+}

internal/grid/types_test.go

@@ -0,0 +1,168 @@
+// Copyright (c) 2015-2023 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package grid
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/tinylib/msgp/msgp"
+)
+
+func TestMarshalUnmarshalMSS(t *testing.T) {
+	v := MSS{"abc": "def", "ghi": "jkl"}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var v2 MSS
+	left, err := v2.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) != 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+	if !reflect.DeepEqual(v, v2) {
+		t.Errorf("MSS: %v != %v", v, v2)
+	}
+}
+
+func TestMarshalUnmarshalMSSNil(t *testing.T) {
+	v := MSS(nil)
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	v2 := MSS(make(map[string]string, 1))
+	left, err := v2.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) != 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+	if !reflect.DeepEqual(v, v2) {
+		t.Errorf("MSS: %v != %v", v, v2)
+	}
+}
+
+func BenchmarkMarshalMsgMSS(b *testing.B) {
+	v := MSS{"abc": "def", "ghi": "jkl"}
+	b.ReportAllocs()
+
+	for b.Loop() {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgMSS(b *testing.B) {
+	v := MSS{"abc": "def", "ghi": "jkl"}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+
+	for b.Loop() {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalMSS(b *testing.B) {
+	v := MSS{"abc": "def", "ghi": "jkl"}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+
+	for b.Loop() {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalBytes(t *testing.T) {
+	v := Bytes([]byte("abc123123123"))
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var v2 Bytes
+	left, err := v2.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) != 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+	if !reflect.DeepEqual(v, v2) {
+		t.Errorf("MSS: %v != %v", v, v2)
+	}
+}
+
+func TestMarshalUnmarshalBytesNil(t *testing.T) {
+	v := Bytes(nil)
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	v2 := Bytes(make([]byte, 1))
+	left, err := v2.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) != 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+	if !reflect.DeepEqual(v, v2) {
+		t.Errorf("MSS: %v != %v", v, v2)
+	}
+}