OP#792 initial commit after fork

internal/bucket/bandwidth/measurement.go

@@ -0,0 +1,92 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package bandwidth
+
+import (
+	"sync"
+	"sync/atomic"
+	"time"
+)
+
+const (
+	// betaBucket is the weight used to calculate exponential moving average
+	betaBucket = 0.1 // Number of averages considered = 1/(1-betaObject)
+)
+
+// bucketMeasurement captures the bandwidth details for one bucket
+type bucketMeasurement struct {
+	lock                 sync.Mutex
+	bytesSinceLastWindow uint64    // Total bytes since last window was processed
+	startTime            time.Time // Start time for window
+	expMovingAvg         float64   // Previously calculate sliding window
+}
+
+// newBucketMeasurement creates a new instance of the measurement with the initial start time.
+func newBucketMeasurement(initTime time.Time) *bucketMeasurement {
+	return &bucketMeasurement{
+		startTime: initTime,
+	}
+}
+
+// incrementBytes add bytes reported for a bucket.
+func (m *bucketMeasurement) incrementBytes(bytes uint64) {
+	atomic.AddUint64(&m.bytesSinceLastWindow, bytes)
+}
+
+// updateExponentialMovingAverage processes the measurements captured so far.
+func (m *bucketMeasurement) updateExponentialMovingAverage(endTime time.Time) {
+	// Calculate aggregate avg bandwidth and exp window avg
+	m.lock.Lock()
+	defer func() {
+		m.startTime = endTime
+		m.lock.Unlock()
+	}()
+
+	if m.startTime.IsZero() {
+		return
+	}
+
+	if endTime.Before(m.startTime) {
+		return
+	}
+
+	duration := endTime.Sub(m.startTime)
+
+	bytesSinceLastWindow := atomic.SwapUint64(&m.bytesSinceLastWindow, 0)
+
+	if m.expMovingAvg == 0 {
+		// Should address initial calculation and should be fine for resuming from 0
+		m.expMovingAvg = float64(bytesSinceLastWindow) / duration.Seconds()
+		return
+	}
+
+	increment := float64(bytesSinceLastWindow) / duration.Seconds()
+	m.expMovingAvg = exponentialMovingAverage(betaBucket, m.expMovingAvg, increment)
+}
+
+// exponentialMovingAverage calculates the exponential moving average
+func exponentialMovingAverage(beta, previousAvg, incrementAvg float64) float64 {
+	return (1-beta)*incrementAvg + beta*previousAvg
+}
+
+// getExpMovingAvgBytesPerSecond returns the exponential moving average for the bucket in bytes
+func (m *bucketMeasurement) getExpMovingAvgBytesPerSecond() float64 {
+	m.lock.Lock()
+	defer m.lock.Unlock()
+	return m.expMovingAvg
+}

internal/bucket/bandwidth/monitor.go

@@ -0,0 +1,215 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package bandwidth
+
+//go:generate msgp -file=$GOFILE -unexported
+
+import (
+	"context"
+	"slices"
+	"sync"
+	"time"
+
+	"golang.org/x/time/rate"
+)
+
+//msgp:ignore bucketThrottle Monitor
+
+type bucketThrottle struct {
+	*rate.Limiter
+	NodeBandwidthPerSec int64
+}
+
+// Monitor holds the state of the global bucket monitor
+type Monitor struct {
+	tlock sync.RWMutex // mutex for bucket throttling
+	mlock sync.RWMutex // mutex for bucket measurement
+
+	bucketsThrottle    map[BucketOptions]*bucketThrottle
+	bucketsMeasurement map[BucketOptions]*bucketMeasurement // Buckets with objects in flight
+
+	bucketMovingAvgTicker *time.Ticker    // Ticker for calculating moving averages
+	ctx                   context.Context // Context for generate
+	NodeCount             uint64
+}
+
+// NewMonitor returns a monitor with defaults.
+func NewMonitor(ctx context.Context, numNodes uint64) *Monitor {
+	m := &Monitor{
+		bucketsMeasurement:    make(map[BucketOptions]*bucketMeasurement),
+		bucketsThrottle:       make(map[BucketOptions]*bucketThrottle),
+		bucketMovingAvgTicker: time.NewTicker(2 * time.Second),
+		ctx:                   ctx,
+		NodeCount:             numNodes,
+	}
+	go m.trackEWMA()
+	return m
+}
+
+func (m *Monitor) updateMeasurement(opts BucketOptions, bytes uint64) {
+	m.mlock.Lock()
+	defer m.mlock.Unlock()
+
+	tm, ok := m.bucketsMeasurement[opts]
+	if !ok {
+		tm = &bucketMeasurement{}
+	}
+	tm.incrementBytes(bytes)
+	m.bucketsMeasurement[opts] = tm
+}
+
+// SelectionFunction for buckets
+type SelectionFunction func(bucket string) bool
+
+// SelectBuckets will select all the buckets passed in.
+func SelectBuckets(buckets ...string) SelectionFunction {
+	if len(buckets) == 0 {
+		return func(bucket string) bool {
+			return true
+		}
+	}
+	return func(bucket string) bool {
+		return slices.Contains(buckets, bucket)
+	}
+}
+
+// Details for the measured bandwidth
+type Details struct {
+	LimitInBytesPerSecond            int64   `json:"limitInBits"`
+	CurrentBandwidthInBytesPerSecond float64 `json:"currentBandwidth"`
+}
+
+// BucketBandwidthReport captures the details for all buckets.
+type BucketBandwidthReport struct {
+	BucketStats map[BucketOptions]Details `json:"bucketStats,omitempty"`
+}
+
+// GetReport gets the report for all bucket bandwidth details.
+func (m *Monitor) GetReport(selectBucket SelectionFunction) *BucketBandwidthReport {
+	m.mlock.RLock()
+	defer m.mlock.RUnlock()
+	return m.getReport(selectBucket)
+}
+
+func (m *Monitor) getReport(selectBucket SelectionFunction) *BucketBandwidthReport {
+	report := &BucketBandwidthReport{
+		BucketStats: make(map[BucketOptions]Details),
+	}
+	for bucketOpts, bucketMeasurement := range m.bucketsMeasurement {
+		if !selectBucket(bucketOpts.Name) {
+			continue
+		}
+		m.tlock.RLock()
+		if tgtThrottle, ok := m.bucketsThrottle[bucketOpts]; ok {
+			currBw := bucketMeasurement.getExpMovingAvgBytesPerSecond()
+			report.BucketStats[bucketOpts] = Details{
+				LimitInBytesPerSecond:            tgtThrottle.NodeBandwidthPerSec * int64(m.NodeCount),
+				CurrentBandwidthInBytesPerSecond: currBw,
+			}
+		}
+		m.tlock.RUnlock()
+	}
+	return report
+}
+
+func (m *Monitor) trackEWMA() {
+	for {
+		select {
+		case <-m.bucketMovingAvgTicker.C:
+			m.updateMovingAvg()
+		case <-m.ctx.Done():
+			return
+		}
+	}
+}
+
+func (m *Monitor) updateMovingAvg() {
+	m.mlock.Lock()
+	defer m.mlock.Unlock()
+	for _, bucketMeasurement := range m.bucketsMeasurement {
+		bucketMeasurement.updateExponentialMovingAverage(time.Now())
+	}
+}
+
+func (m *Monitor) init(opts BucketOptions) {
+	m.mlock.Lock()
+	defer m.mlock.Unlock()
+
+	_, ok := m.bucketsMeasurement[opts]
+	if !ok {
+		m.bucketsMeasurement[opts] = newBucketMeasurement(time.Now())
+	}
+}
+
+// DeleteBucket deletes monitoring the 'bucket'
+func (m *Monitor) DeleteBucket(bucket string) {
+	m.tlock.Lock()
+	for opts := range m.bucketsThrottle {
+		if opts.Name == bucket {
+			delete(m.bucketsThrottle, opts)
+		}
+	}
+	m.tlock.Unlock()
+
+	m.mlock.Lock()
+	for opts := range m.bucketsMeasurement {
+		if opts.Name == bucket {
+			delete(m.bucketsMeasurement, opts)
+		}
+	}
+	m.mlock.Unlock()
+}
+
+// DeleteBucketThrottle deletes monitoring for a bucket's target
+func (m *Monitor) DeleteBucketThrottle(bucket, arn string) {
+	m.tlock.Lock()
+	delete(m.bucketsThrottle, BucketOptions{Name: bucket, ReplicationARN: arn})
+	m.tlock.Unlock()
+	m.mlock.Lock()
+	delete(m.bucketsMeasurement, BucketOptions{Name: bucket, ReplicationARN: arn})
+	m.mlock.Unlock()
+}
+
+// throttle returns currently configured throttle for this bucket
+func (m *Monitor) throttle(opts BucketOptions) *bucketThrottle {
+	m.tlock.RLock()
+	defer m.tlock.RUnlock()
+	return m.bucketsThrottle[opts]
+}
+
+// SetBandwidthLimit sets the bandwidth limit for a bucket
+func (m *Monitor) SetBandwidthLimit(bucket, arn string, limit int64) {
+	m.tlock.Lock()
+	defer m.tlock.Unlock()
+	limitBytes := limit / int64(m.NodeCount)
+	throttle, ok := m.bucketsThrottle[BucketOptions{Name: bucket, ReplicationARN: arn}]
+	if !ok {
+		throttle = &bucketThrottle{}
+	}
+	throttle.NodeBandwidthPerSec = limitBytes
+	throttle.Limiter = rate.NewLimiter(rate.Limit(float64(limitBytes)), int(limitBytes))
+	m.bucketsThrottle[BucketOptions{Name: bucket, ReplicationARN: arn}] = throttle
+}
+
+// IsThrottled returns true if a bucket has bandwidth throttling enabled.
+func (m *Monitor) IsThrottled(bucket, arn string) bool {
+	m.tlock.RLock()
+	defer m.tlock.RUnlock()
+	_, ok := m.bucketsThrottle[BucketOptions{Name: bucket, ReplicationARN: arn}]
+	return ok
+}

internal/bucket/bandwidth/monitor_gen.go

@@ -0,0 +1,220 @@
+// Code generated by github.com/tinylib/msgp DO NOT EDIT.
+
+package bandwidth
+
+import (
+	"github.com/tinylib/msgp/msgp"
+)
+
+// DecodeMsg implements msgp.Decodable
+func (z *BucketBandwidthReport) DecodeMsg(dc *msgp.Reader) (err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, err = dc.ReadMapHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, err = dc.ReadMapKeyPtr()
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		default:
+			err = dc.Skip()
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z BucketBandwidthReport) EncodeMsg(en *msgp.Writer) (err error) {
+	// map header, size 0
+	_ = z
+	err = en.Append(0x80)
+	if err != nil {
+		return
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z BucketBandwidthReport) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// map header, size 0
+	_ = z
+	o = append(o, 0x80)
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *BucketBandwidthReport) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z BucketBandwidthReport) Msgsize() (s int) {
+	s = 1
+	return
+}
+
+// DecodeMsg implements msgp.Decodable
+func (z *Details) DecodeMsg(dc *msgp.Reader) (err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, err = dc.ReadMapHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, err = dc.ReadMapKeyPtr()
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "LimitInBytesPerSecond":
+			z.LimitInBytesPerSecond, err = dc.ReadInt64()
+			if err != nil {
+				err = msgp.WrapError(err, "LimitInBytesPerSecond")
+				return
+			}
+		case "CurrentBandwidthInBytesPerSecond":
+			z.CurrentBandwidthInBytesPerSecond, err = dc.ReadFloat64()
+			if err != nil {
+				err = msgp.WrapError(err, "CurrentBandwidthInBytesPerSecond")
+				return
+			}
+		default:
+			err = dc.Skip()
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z Details) EncodeMsg(en *msgp.Writer) (err error) {
+	// map header, size 2
+	// write "LimitInBytesPerSecond"
+	err = en.Append(0x82, 0xb5, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x49, 0x6e, 0x42, 0x79, 0x74, 0x65, 0x73, 0x50, 0x65, 0x72, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64)
+	if err != nil {
+		return
+	}
+	err = en.WriteInt64(z.LimitInBytesPerSecond)
+	if err != nil {
+		err = msgp.WrapError(err, "LimitInBytesPerSecond")
+		return
+	}
+	// write "CurrentBandwidthInBytesPerSecond"
+	err = en.Append(0xd9, 0x20, 0x43, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x42, 0x61, 0x6e, 0x64, 0x77, 0x69, 0x64, 0x74, 0x68, 0x49, 0x6e, 0x42, 0x79, 0x74, 0x65, 0x73, 0x50, 0x65, 0x72, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64)
+	if err != nil {
+		return
+	}
+	err = en.WriteFloat64(z.CurrentBandwidthInBytesPerSecond)
+	if err != nil {
+		err = msgp.WrapError(err, "CurrentBandwidthInBytesPerSecond")
+		return
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z Details) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// map header, size 2
+	// string "LimitInBytesPerSecond"
+	o = append(o, 0x82, 0xb5, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x49, 0x6e, 0x42, 0x79, 0x74, 0x65, 0x73, 0x50, 0x65, 0x72, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64)
+	o = msgp.AppendInt64(o, z.LimitInBytesPerSecond)
+	// string "CurrentBandwidthInBytesPerSecond"
+	o = append(o, 0xd9, 0x20, 0x43, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x42, 0x61, 0x6e, 0x64, 0x77, 0x69, 0x64, 0x74, 0x68, 0x49, 0x6e, 0x42, 0x79, 0x74, 0x65, 0x73, 0x50, 0x65, 0x72, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64)
+	o = msgp.AppendFloat64(o, z.CurrentBandwidthInBytesPerSecond)
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *Details) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "LimitInBytesPerSecond":
+			z.LimitInBytesPerSecond, bts, err = msgp.ReadInt64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "LimitInBytesPerSecond")
+				return
+			}
+		case "CurrentBandwidthInBytesPerSecond":
+			z.CurrentBandwidthInBytesPerSecond, bts, err = msgp.ReadFloat64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "CurrentBandwidthInBytesPerSecond")
+				return
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z Details) Msgsize() (s int) {
+	s = 1 + 22 + msgp.Int64Size + 34 + msgp.Float64Size
+	return
+}

internal/bucket/bandwidth/monitor_gen_test.go

@@ -0,0 +1,236 @@
+// Code generated by github.com/tinylib/msgp DO NOT EDIT.
+
+package bandwidth
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/tinylib/msgp/msgp"
+)
+
+func TestMarshalUnmarshalBucketBandwidthReport(t *testing.T) {
+	v := BucketBandwidthReport{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgBucketBandwidthReport(b *testing.B) {
+	v := BucketBandwidthReport{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgBucketBandwidthReport(b *testing.B) {
+	v := BucketBandwidthReport{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalBucketBandwidthReport(b *testing.B) {
+	v := BucketBandwidthReport{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeBucketBandwidthReport(t *testing.T) {
+	v := BucketBandwidthReport{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeBucketBandwidthReport Msgsize() is inaccurate")
+	}
+
+	vn := BucketBandwidthReport{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeBucketBandwidthReport(b *testing.B) {
+	v := BucketBandwidthReport{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeBucketBandwidthReport(b *testing.B) {
+	v := BucketBandwidthReport{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalDetails(t *testing.T) {
+	v := Details{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgDetails(b *testing.B) {
+	v := Details{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgDetails(b *testing.B) {
+	v := Details{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalDetails(b *testing.B) {
+	v := Details{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeDetails(t *testing.T) {
+	v := Details{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeDetails Msgsize() is inaccurate")
+	}
+
+	vn := Details{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeDetails(b *testing.B) {
+	v := Details{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeDetails(b *testing.B) {
+	v := Details{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}

internal/bucket/bandwidth/monitor_test.go

@@ -0,0 +1,127 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package bandwidth
+
+import (
+	"reflect"
+	"testing"
+	"time"
+)
+
+const (
+	oneMiB uint64 = 1024 * 1024
+)
+
+func TestMonitor_GetReport(t *testing.T) {
+	type fields struct {
+		activeBuckets map[BucketOptions]*bucketMeasurement
+		endTime       time.Time
+		update2       uint64
+		endTime2      time.Time
+	}
+	start := time.Now()
+	m0 := newBucketMeasurement(start)
+	m0.incrementBytes(0)
+	m1MiBPS := newBucketMeasurement(start)
+	m1MiBPS.incrementBytes(oneMiB)
+
+	test1Want := make(map[BucketOptions]Details)
+	test1Want[BucketOptions{Name: "bucket", ReplicationARN: "arn"}] = Details{LimitInBytesPerSecond: 1024 * 1024, CurrentBandwidthInBytesPerSecond: 0}
+	test1Want2 := make(map[BucketOptions]Details)
+	test1Want2[BucketOptions{Name: "bucket", ReplicationARN: "arn"}] = Details{
+		LimitInBytesPerSecond:            1024 * 1024,
+		CurrentBandwidthInBytesPerSecond: (1024 * 1024) / start.Add(2*time.Second).Sub(start.Add(1*time.Second)).Seconds(),
+	}
+
+	test2Want := make(map[BucketOptions]Details)
+	test2Want[BucketOptions{Name: "bucket", ReplicationARN: "arn"}] = Details{LimitInBytesPerSecond: 1024 * 1024, CurrentBandwidthInBytesPerSecond: float64(oneMiB)}
+	test2Want2 := make(map[BucketOptions]Details)
+	test2Want2[BucketOptions{Name: "bucket", ReplicationARN: "arn"}] = Details{
+		LimitInBytesPerSecond:            1024 * 1024,
+		CurrentBandwidthInBytesPerSecond: exponentialMovingAverage(betaBucket, float64(oneMiB), 2*float64(oneMiB)),
+	}
+
+	test1ActiveBuckets := make(map[BucketOptions]*bucketMeasurement)
+	test1ActiveBuckets[BucketOptions{Name: "bucket", ReplicationARN: "arn"}] = m0
+	test1ActiveBuckets2 := make(map[BucketOptions]*bucketMeasurement)
+	test1ActiveBuckets2[BucketOptions{Name: "bucket", ReplicationARN: "arn"}] = m1MiBPS
+
+	tests := []struct {
+		name   string
+		fields fields
+		want   *BucketBandwidthReport
+		want2  *BucketBandwidthReport
+	}{
+		{
+			name: "ZeroToOne",
+			fields: fields{
+				activeBuckets: test1ActiveBuckets,
+				endTime:       start.Add(1 * time.Second),
+				update2:       oneMiB,
+				endTime2:      start.Add(2 * time.Second),
+			},
+			want: &BucketBandwidthReport{
+				BucketStats: test1Want,
+			},
+			want2: &BucketBandwidthReport{
+				BucketStats: test1Want2,
+			},
+		},
+		{
+			name: "OneToTwo",
+			fields: fields{
+				activeBuckets: test1ActiveBuckets2,
+				endTime:       start.Add(1 * time.Second),
+				update2:       2 * oneMiB,
+				endTime2:      start.Add(2 * time.Second),
+			},
+			want: &BucketBandwidthReport{
+				BucketStats: test2Want,
+			},
+			want2: &BucketBandwidthReport{
+				BucketStats: test2Want2,
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+			thr := bucketThrottle{
+				NodeBandwidthPerSec: 1024 * 1024,
+			}
+			th := make(map[BucketOptions]*bucketThrottle)
+			th[BucketOptions{Name: "bucket", ReplicationARN: "arn"}] = &thr
+			m := &Monitor{
+				bucketsMeasurement: tt.fields.activeBuckets,
+				bucketsThrottle:    th,
+				NodeCount:          1,
+			}
+			m.bucketsMeasurement[BucketOptions{Name: "bucket", ReplicationARN: "arn"}].updateExponentialMovingAverage(tt.fields.endTime)
+			got := m.GetReport(SelectBuckets())
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("GetReport() = %v, want %v", got, tt.want)
+			}
+			m.bucketsMeasurement[BucketOptions{Name: "bucket", ReplicationARN: "arn"}].incrementBytes(tt.fields.update2)
+			m.bucketsMeasurement[BucketOptions{Name: "bucket", ReplicationARN: "arn"}].updateExponentialMovingAverage(tt.fields.endTime2)
+			got = m.GetReport(SelectBuckets())
+			if !reflect.DeepEqual(got.BucketStats, tt.want2.BucketStats) {
+				t.Errorf("GetReport() = %v, want %v", got.BucketStats, tt.want2.BucketStats)
+			}
+		})
+	}
+}

internal/bucket/bandwidth/reader.go

@@ -0,0 +1,107 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package bandwidth
+
+import (
+	"context"
+	"io"
+	"math"
+)
+
+// MonitoredReader represents a throttled reader subject to bandwidth monitoring
+type MonitoredReader struct {
+	r        io.Reader
+	throttle *bucketThrottle
+	ctx      context.Context // request context
+	lastErr  error           // last error reported, if this non-nil all reads will fail.
+	m        *Monitor
+	opts     *MonitorReaderOptions
+}
+
+// BucketOptions represents the bucket and optionally its replication target pair.
+type BucketOptions struct {
+	Name           string
+	ReplicationARN string // This is optional, and not mandatory.
+}
+
+// MonitorReaderOptions provides configurable options for monitor reader implementation.
+type MonitorReaderOptions struct {
+	BucketOptions
+	HeaderSize int
+}
+
+// Read implements a throttled read
+func (r *MonitoredReader) Read(buf []byte) (n int, err error) {
+	if r.throttle == nil {
+		return r.r.Read(buf)
+	}
+	if r.lastErr != nil {
+		err = r.lastErr
+		return n, err
+	}
+	b := r.throttle.Burst()  // maximum available tokens
+	need := len(buf)         // number of bytes requested by caller
+	hdr := r.opts.HeaderSize // remaining header bytes
+	var tokens int           // number of tokens to request
+
+	if hdr > 0 { // available tokens go towards header first
+		if hdr < b { // all of header can be accommodated
+			r.opts.HeaderSize = 0
+			need = int(math.Min(float64(b-hdr), float64(need))) // use remaining tokens towards payload
+			tokens = need + hdr
+		} else { // part of header can be accommodated
+			r.opts.HeaderSize -= b - 1
+			need = 1 // to ensure we read at least one byte for every Read
+			tokens = b
+		}
+	} else { // all tokens go towards payload
+		need = int(math.Min(float64(b), float64(need)))
+		tokens = need
+	}
+	// reduce tokens requested according to availability
+	av := int(r.throttle.Tokens())
+	if av < tokens && av > 0 {
+		tokens = av
+		need = int(math.Min(float64(tokens), float64(need)))
+	}
+	err = r.throttle.WaitN(r.ctx, tokens)
+	if err != nil {
+		return n, err
+	}
+	n, err = r.r.Read(buf[:need])
+	if err != nil {
+		r.lastErr = err
+		return n, err
+	}
+	r.m.updateMeasurement(r.opts.BucketOptions, uint64(tokens))
+	return n, err
+}
+
+// NewMonitoredReader returns reference to a monitored reader that throttles reads to configured bandwidth for the
+// bucket.
+func NewMonitoredReader(ctx context.Context, m *Monitor, r io.Reader, opts *MonitorReaderOptions) *MonitoredReader {
+	reader := MonitoredReader{
+		r:        r,
+		throttle: m.throttle(opts.BucketOptions),
+		m:        m,
+		opts:     opts,
+		ctx:      ctx,
+	}
+	reader.m.init(opts.BucketOptions)
+	return &reader
+}