OP#792 initial commit after fork

internal/dsync/.gitignore

@@ -0,0 +1,3 @@
+dsync.test
+coverage.txt
+*.out

internal/dsync/drwmutex.go

@@ -0,0 +1,735 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package dsync
+
+import (
+	"context"
+	"errors"
+	"math/rand"
+	"slices"
+	"sort"
+	"strconv"
+	"sync"
+	"time"
+
+	xioutil "github.com/minio/minio/internal/ioutil"
+	"github.com/minio/minio/internal/mcontext"
+	"github.com/minio/pkg/v3/console"
+	"github.com/minio/pkg/v3/env"
+)
+
+// Indicator if logging is enabled.
+var dsyncLog bool
+
+// Retry unit interval
+var lockRetryMinInterval time.Duration
+
+var lockRetryBackOff func(*rand.Rand, uint) time.Duration
+
+func init() {
+	// Check for MINIO_DSYNC_TRACE env variable, if set logging will be enabled for failed REST operations.
+	dsyncLog = env.Get("_MINIO_DSYNC_TRACE", "0") == "1"
+
+	lockRetryMinInterval = 250 * time.Millisecond
+	if lri := env.Get("_MINIO_LOCK_RETRY_INTERVAL", ""); lri != "" {
+		v, err := strconv.Atoi(lri)
+		if err != nil {
+			panic(err)
+		}
+		lockRetryMinInterval = time.Duration(v) * time.Millisecond
+	}
+
+	lockRetryBackOff = backoffWait(
+		lockRetryMinInterval,
+		100*time.Millisecond,
+		5*time.Second,
+	)
+}
+
+func log(format string, data ...any) {
+	if dsyncLog {
+		console.Printf(format, data...)
+	}
+}
+
+const (
+	// dRWMutexAcquireTimeout - default tolerance limit to wait for lock acquisition before.
+	drwMutexAcquireTimeout = 1 * time.Second // 1 second.
+
+	// dRWMutexRefreshTimeout - default timeout for the refresh call
+	drwMutexRefreshCallTimeout = 5 * time.Second
+
+	// dRWMutexUnlockTimeout - default timeout for the unlock call
+	drwMutexUnlockCallTimeout = 30 * time.Second
+
+	// dRWMutexForceUnlockTimeout - default timeout for the unlock call
+	drwMutexForceUnlockCallTimeout = 30 * time.Second
+
+	// dRWMutexRefreshInterval - default the interval between two refresh calls
+	drwMutexRefreshInterval = 10 * time.Second
+
+	drwMutexInfinite = 1<<63 - 1
+)
+
+// Timeouts are timeouts for specific operations.
+type Timeouts struct {
+	// Acquire - tolerance limit to wait for lock acquisition before.
+	Acquire time.Duration
+
+	// RefreshCall - timeout for the refresh call
+	RefreshCall time.Duration
+
+	// UnlockCall - timeout for the unlock call
+	UnlockCall time.Duration
+
+	// ForceUnlockCall - timeout for the force unlock call
+	ForceUnlockCall time.Duration
+}
+
+// DefaultTimeouts contains default timeouts.
+var DefaultTimeouts = Timeouts{
+	Acquire:         drwMutexAcquireTimeout,
+	RefreshCall:     drwMutexRefreshCallTimeout,
+	UnlockCall:      drwMutexUnlockCallTimeout,
+	ForceUnlockCall: drwMutexForceUnlockCallTimeout,
+}
+
+// A DRWMutex is a distributed mutual exclusion lock.
+type DRWMutex struct {
+	Names                []string
+	writeLocks           []string // Array of nodes that granted a write lock
+	readLocks            []string // Array of array of nodes that granted reader locks
+	rng                  *rand.Rand
+	m                    sync.Mutex // Mutex to prevent multiple simultaneous locks from this node
+	clnt                 *Dsync
+	cancelRefresh        context.CancelFunc
+	refreshInterval      time.Duration
+	lockRetryMinInterval time.Duration
+}
+
+// Granted - represents a structure of a granted lock.
+type Granted struct {
+	index   int
+	lockUID string // Locked if set with UID string, unlocked if empty
+}
+
+func (g *Granted) isLocked() bool {
+	return isLocked(g.lockUID)
+}
+
+func isLocked(uid string) bool {
+	return len(uid) > 0
+}
+
+// NewDRWMutex - initializes a new dsync RW mutex.
+func NewDRWMutex(clnt *Dsync, names ...string) *DRWMutex {
+	restClnts, _ := clnt.GetLockers()
+	sort.Strings(names)
+	return &DRWMutex{
+		writeLocks:           make([]string, len(restClnts)),
+		readLocks:            make([]string, len(restClnts)),
+		Names:                names,
+		clnt:                 clnt,
+		rng:                  rand.New(&lockedRandSource{src: rand.NewSource(time.Now().UTC().UnixNano())}),
+		refreshInterval:      drwMutexRefreshInterval,
+		lockRetryMinInterval: lockRetryMinInterval,
+	}
+}
+
+// Lock holds a write lock on dm.
+//
+// If the lock is already in use, the calling go routine
+// blocks until the mutex is available.
+func (dm *DRWMutex) Lock(id, source string) {
+	isReadLock := false
+	dm.lockBlocking(context.Background(), nil, id, source, isReadLock, Options{
+		Timeout: drwMutexInfinite,
+	})
+}
+
+// Options lock options.
+type Options struct {
+	Timeout       time.Duration
+	RetryInterval time.Duration
+}
+
+// GetLock tries to get a write lock on dm before the timeout elapses.
+//
+// If the lock is already in use, the calling go routine
+// blocks until either the mutex becomes available and return success or
+// more time has passed than the timeout value and return false.
+func (dm *DRWMutex) GetLock(ctx context.Context, cancel context.CancelFunc, id, source string, opts Options) (locked bool) {
+	isReadLock := false
+	return dm.lockBlocking(ctx, cancel, id, source, isReadLock, opts)
+}
+
+// RLock holds a read lock on dm.
+//
+// If one or more read locks are already in use, it will grant another lock.
+// Otherwise the calling go routine blocks until the mutex is available.
+func (dm *DRWMutex) RLock(id, source string) {
+	isReadLock := true
+	dm.lockBlocking(context.Background(), nil, id, source, isReadLock, Options{
+		Timeout: drwMutexInfinite,
+	})
+}
+
+// GetRLock tries to get a read lock on dm before the timeout elapses.
+//
+// If one or more read locks are already in use, it will grant another lock.
+// Otherwise the calling go routine blocks until either the mutex becomes
+// available and return success or more time has passed than the timeout
+// value and return false.
+func (dm *DRWMutex) GetRLock(ctx context.Context, cancel context.CancelFunc, id, source string, opts Options) (locked bool) {
+	isReadLock := true
+	return dm.lockBlocking(ctx, cancel, id, source, isReadLock, opts)
+}
+
+// lockBlocking will try to acquire either a read or a write lock
+//
+// The function will loop using a built-in timing randomized back-off
+// algorithm until either the lock is acquired successfully or more
+// time has elapsed than the timeout value.
+func (dm *DRWMutex) lockBlocking(ctx context.Context, lockLossCallback func(), id, source string, isReadLock bool, opts Options) (locked bool) {
+	restClnts, _ := dm.clnt.GetLockers()
+
+	// Create lock array to capture the successful lockers
+	locks := make([]string, len(restClnts))
+
+	// Add total timeout
+	ctx, cancel := context.WithTimeout(ctx, opts.Timeout)
+	defer cancel()
+
+	// Tolerance is not set, defaults to half of the locker clients.
+	tolerance := len(restClnts) / 2
+
+	// Quorum is effectively = total clients subtracted with tolerance limit
+	quorum := len(restClnts) - tolerance
+	if !isReadLock {
+		// In situations for write locks, as a special case
+		// to avoid split brains we make sure to acquire
+		// quorum + 1 when tolerance is exactly half of the
+		// total locker clients.
+		if quorum == tolerance {
+			quorum++
+		}
+	}
+
+	log("lockBlocking %s/%s for %#v: lockType readLock(%t), additional opts: %#v, quorum: %d, tolerance: %d, lockClients: %d\n", id, source, dm.Names, isReadLock, opts, quorum, tolerance, len(restClnts))
+
+	tolerance = len(restClnts) - quorum
+	attempt := uint(0)
+
+	for {
+		select {
+		case <-ctx.Done():
+			return false
+		default:
+			// Try to acquire the lock.
+			if locked = lock(ctx, dm.clnt, &locks, id, source, isReadLock, tolerance, quorum, dm.Names...); locked {
+				dm.m.Lock()
+
+				// If success, copy array to object
+				if isReadLock {
+					copy(dm.readLocks, locks)
+				} else {
+					copy(dm.writeLocks, locks)
+				}
+
+				dm.m.Unlock()
+				log("lockBlocking %s/%s for %#v: granted\n", id, source, dm.Names)
+
+				// Refresh lock continuously and cancel if there is no quorum in the lock anymore
+				dm.startContinuousLockRefresh(lockLossCallback, id, source, quorum)
+
+				return locked
+			}
+
+			switch {
+			case opts.RetryInterval < 0:
+				return false
+			case opts.RetryInterval > 0:
+				time.Sleep(opts.RetryInterval)
+			default:
+				attempt++
+				time.Sleep(lockRetryBackOff(dm.rng, attempt))
+			}
+		}
+	}
+}
+
+func (dm *DRWMutex) startContinuousLockRefresh(lockLossCallback func(), id, source string, quorum int) {
+	ctx, cancel := context.WithCancel(context.Background())
+
+	dm.m.Lock()
+	dm.cancelRefresh = cancel
+	dm.m.Unlock()
+
+	go func() {
+		defer cancel()
+
+		refreshTimer := time.NewTimer(dm.refreshInterval)
+		defer refreshTimer.Stop()
+
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-refreshTimer.C:
+				noQuorum, err := refreshLock(ctx, dm.clnt, id, source, quorum)
+				if err == nil && noQuorum {
+					// Clean the lock locally and in remote nodes
+					forceUnlock(ctx, dm.clnt, id)
+					// Execute the caller lock loss callback
+					if lockLossCallback != nil {
+						lockLossCallback()
+					}
+					return
+				}
+
+				refreshTimer.Reset(dm.refreshInterval)
+			}
+		}
+	}()
+}
+
+func forceUnlock(ctx context.Context, ds *Dsync, id string) {
+	ctx, cancel := context.WithTimeout(ctx, ds.Timeouts.ForceUnlockCall)
+	defer cancel()
+
+	restClnts, _ := ds.GetLockers()
+
+	args := LockArgs{
+		UID: id,
+	}
+
+	var wg sync.WaitGroup
+	for index, c := range restClnts {
+		wg.Add(1)
+		// Send refresh request to all nodes
+		go func(index int, c NetLocker) {
+			defer wg.Done()
+			c.ForceUnlock(ctx, args)
+		}(index, c)
+	}
+	wg.Wait()
+}
+
+type refreshResult struct {
+	offline   bool
+	refreshed bool
+}
+
+// Refresh the given lock in all nodes, return true to indicate if a lock
+// does not exist in enough quorum nodes.
+func refreshLock(ctx context.Context, ds *Dsync, id, source string, quorum int) (bool, error) {
+	restClnts, _ := ds.GetLockers()
+
+	// Create buffered channel of size equal to total number of nodes.
+	ch := make(chan refreshResult, len(restClnts))
+	var wg sync.WaitGroup
+
+	args := LockArgs{
+		UID: id,
+	}
+
+	for index, c := range restClnts {
+		wg.Add(1)
+		// Send refresh request to all nodes
+		go func(index int, c NetLocker) {
+			defer wg.Done()
+
+			if c == nil {
+				ch <- refreshResult{offline: true}
+				return
+			}
+
+			ctx, cancel := context.WithTimeout(ctx, ds.Timeouts.RefreshCall)
+			defer cancel()
+
+			refreshed, err := c.Refresh(ctx, args)
+			if err != nil {
+				ch <- refreshResult{offline: true}
+				log("dsync: Unable to call Refresh failed with %s for %#v at %s\n", err, args, c)
+			} else {
+				ch <- refreshResult{refreshed: refreshed}
+				log("dsync: Refresh returned false for %#v at %s\n", args, c)
+			}
+		}(index, c)
+	}
+
+	// Wait until we have either
+	//
+	// a) received all refresh responses
+	// b) received too many refreshed for quorum to be still possible
+	// c) timed out
+	//
+	lockNotFound, lockRefreshed := 0, 0
+	done := false
+
+	for range len(restClnts) {
+		select {
+		case refreshResult := <-ch:
+			if refreshResult.offline {
+				continue
+			}
+			if refreshResult.refreshed {
+				lockRefreshed++
+			} else {
+				lockNotFound++
+			}
+			if lockRefreshed >= quorum || lockNotFound > len(restClnts)-quorum {
+				done = true
+			}
+		case <-ctx.Done():
+			// Refreshing is canceled
+			return false, ctx.Err()
+		}
+		if done {
+			break
+		}
+	}
+
+	// We may have some unused results in ch, release them async.
+	go func() {
+		wg.Wait()
+		xioutil.SafeClose(ch)
+		for range ch {
+		}
+	}()
+
+	noQuorum := lockNotFound > len(restClnts)-quorum
+	return noQuorum, nil
+}
+
+// lock tries to acquire the distributed lock, returning true or false.
+func lock(ctx context.Context, ds *Dsync, locks *[]string, id, source string, isReadLock bool, tolerance, quorum int, names ...string) bool {
+	for i := range *locks {
+		(*locks)[i] = ""
+	}
+
+	restClnts, owner := ds.GetLockers()
+
+	// Create buffered channel of size equal to total number of nodes.
+	ch := make(chan Granted, len(restClnts))
+	var wg sync.WaitGroup
+
+	args := LockArgs{
+		Owner:     owner,
+		UID:       id,
+		Resources: names,
+		Source:    source,
+		Quorum:    &quorum,
+	}
+
+	// Combined timeout for the lock attempt.
+	ctx, cancel := context.WithTimeout(ctx, ds.Timeouts.Acquire)
+	defer cancel()
+
+	// Special context for NetLockers - do not use timeouts.
+	// Also, pass the trace context info if found for debugging
+	netLockCtx := context.Background()
+
+	tc, ok := ctx.Value(mcontext.ContextTraceKey).(*mcontext.TraceCtxt)
+	if ok {
+		netLockCtx = context.WithValue(netLockCtx, mcontext.ContextTraceKey, tc)
+	}
+
+	for index, c := range restClnts {
+		wg.Add(1)
+		// broadcast lock request to all nodes
+		go func(index int, isReadLock bool, c NetLocker) {
+			defer wg.Done()
+
+			g := Granted{index: index}
+			if c == nil {
+				log("dsync: nil locker\n")
+				ch <- g
+				return
+			}
+
+			var locked bool
+			var err error
+			if isReadLock {
+				if locked, err = c.RLock(netLockCtx, args); err != nil {
+					log("dsync: Unable to call RLock failed with %s for %#v at %s\n", err, args, c)
+				}
+			} else {
+				if locked, err = c.Lock(netLockCtx, args); err != nil {
+					log("dsync: Unable to call Lock failed with %s for %#v at %s\n", err, args, c)
+				}
+			}
+			if locked {
+				g.lockUID = args.UID
+			}
+			ch <- g
+		}(index, isReadLock, c)
+	}
+
+	// Wait until we have either
+	//
+	// a) received all lock responses
+	// b) received too many 'non-'locks for quorum to be still possible
+	// c) timed out
+	//
+	i, locksFailed := 0, 0
+	done := false
+
+	for ; i < len(restClnts); i++ { // Loop until we acquired all locks
+		select {
+		case grant := <-ch:
+			if grant.isLocked() {
+				// Mark that this node has acquired the lock
+				(*locks)[grant.index] = grant.lockUID
+			} else {
+				locksFailed++
+				if locksFailed > tolerance {
+					// We know that we are not going to get the lock anymore,
+					// so exit out and release any locks that did get acquired
+					done = true
+				}
+			}
+		case <-ctx.Done():
+			// Capture timedout locks as failed or took too long
+			locksFailed++
+			if locksFailed > tolerance {
+				// We know that we are not going to get the lock anymore,
+				// so exit out and release any locks that did get acquired
+				done = true
+			}
+		}
+
+		if done {
+			break
+		}
+	}
+
+	quorumLocked := checkQuorumLocked(locks, quorum) && locksFailed <= tolerance
+	if !quorumLocked {
+		log("dsync: Unable to acquire lock in quorum %#v\n", args)
+		// Release all acquired locks without quorum.
+		if !releaseAll(ctx, ds, tolerance, owner, locks, isReadLock, restClnts, names...) {
+			log("Unable to release acquired locks, these locks will expire automatically %#v\n", args)
+		}
+	}
+
+	// We may have some unused results in ch, release them async.
+	go func() {
+		wg.Wait()
+		xioutil.SafeClose(ch)
+		for grantToBeReleased := range ch {
+			if grantToBeReleased.isLocked() {
+				// release abandoned lock
+				log("Releasing abandoned lock\n")
+				sendRelease(ctx, ds, restClnts[grantToBeReleased.index],
+					owner, grantToBeReleased.lockUID, isReadLock, names...)
+			}
+		}
+	}()
+
+	return quorumLocked
+}
+
+// checkFailedUnlocks determines whether we have sufficiently unlocked all
+// resources to ensure no deadlocks for future callers
+func checkFailedUnlocks(locks []string, tolerance int) bool {
+	unlocksFailed := 0
+	for lockID := range locks {
+		if isLocked(locks[lockID]) {
+			unlocksFailed++
+		}
+	}
+
+	// Unlock failures are higher than tolerance limit
+	// for this instance of unlocker, we should let the
+	// caller know that lock is not successfully released
+	// yet.
+	if len(locks)-tolerance == tolerance {
+		// In case of split brain scenarios where
+		// tolerance is exactly half of the len(*locks)
+		// then we need to make sure we have unlocked
+		// upto tolerance+1 - especially for RUnlock
+		// to ensure that we don't end up with active
+		// read locks on the resource after unlocking
+		// only half of the lockers.
+		return unlocksFailed >= tolerance
+	}
+	return unlocksFailed > tolerance
+}
+
+// checkQuorumLocked determines whether we have locked the required quorum of underlying locks or not
+func checkQuorumLocked(locks *[]string, quorum int) bool {
+	count := 0
+	for _, uid := range *locks {
+		if isLocked(uid) {
+			count++
+		}
+	}
+
+	return count >= quorum
+}
+
+// releaseAll releases all locks that are marked as locked
+func releaseAll(ctx context.Context, ds *Dsync, tolerance int, owner string, locks *[]string, isReadLock bool, restClnts []NetLocker, names ...string) bool {
+	var wg sync.WaitGroup
+	for lockID := range restClnts {
+		wg.Add(1)
+		go func(lockID int) {
+			defer wg.Done()
+			if sendRelease(ctx, ds, restClnts[lockID], owner, (*locks)[lockID], isReadLock, names...) {
+				(*locks)[lockID] = ""
+			}
+		}(lockID)
+	}
+	wg.Wait()
+
+	// Return true if releaseAll was successful, otherwise we return 'false'
+	// to indicate we haven't sufficiently unlocked lockers to avoid deadlocks.
+	//
+	// Caller may use this as an indication to call again.
+	return !checkFailedUnlocks(*locks, tolerance)
+}
+
+// Unlock unlocks the write lock.
+//
+// It is a run-time error if dm is not locked on entry to Unlock.
+func (dm *DRWMutex) Unlock(ctx context.Context) {
+	dm.m.Lock()
+	dm.cancelRefresh()
+	dm.m.Unlock()
+
+	restClnts, owner := dm.clnt.GetLockers()
+	// create temp array on stack
+	locks := make([]string, len(restClnts))
+
+	{
+		dm.m.Lock()
+		defer dm.m.Unlock()
+
+		// Check if minimally a single bool is set in the writeLocks array
+		lockFound := slices.ContainsFunc(dm.writeLocks, isLocked)
+		if !lockFound {
+			panic("Trying to Unlock() while no Lock() is active")
+		}
+
+		// Copy write locks to stack array
+		copy(locks, dm.writeLocks)
+	}
+
+	// Tolerance is not set, defaults to half of the locker clients.
+	tolerance := len(restClnts) / 2
+
+	isReadLock := false
+	started := time.Now()
+	// Do async unlocking.
+	// This means unlock will no longer block on the network or missing quorum.
+	go func() {
+		ctx, done := context.WithTimeout(ctx, drwMutexUnlockCallTimeout)
+		defer done()
+		for !releaseAll(ctx, dm.clnt, tolerance, owner, &locks, isReadLock, restClnts, dm.Names...) {
+			time.Sleep(time.Duration(dm.rng.Float64() * float64(dm.lockRetryMinInterval)))
+			if time.Since(started) > dm.clnt.Timeouts.UnlockCall {
+				return
+			}
+		}
+	}()
+}
+
+// RUnlock releases a read lock held on dm.
+//
+// It is a run-time error if dm is not locked on entry to RUnlock.
+func (dm *DRWMutex) RUnlock(ctx context.Context) {
+	dm.m.Lock()
+	dm.cancelRefresh()
+	dm.m.Unlock()
+
+	restClnts, owner := dm.clnt.GetLockers()
+	// create temp array on stack
+	locks := make([]string, len(restClnts))
+
+	{
+		dm.m.Lock()
+		defer dm.m.Unlock()
+
+		// Check if minimally a single bool is set in the writeLocks array
+		lockFound := slices.ContainsFunc(dm.readLocks, isLocked)
+		if !lockFound {
+			panic("Trying to RUnlock() while no RLock() is active")
+		}
+
+		// Copy write locks to stack array
+		copy(locks, dm.readLocks)
+	}
+
+	// Tolerance is not set, defaults to half of the locker clients.
+	tolerance := len(restClnts) / 2
+	isReadLock := true
+	started := time.Now()
+	// Do async unlocking.
+	// This means unlock will no longer block on the network or missing quorum.
+	go func() {
+		for !releaseAll(ctx, dm.clnt, tolerance, owner, &locks, isReadLock, restClnts, dm.Names...) {
+			time.Sleep(time.Duration(dm.rng.Float64() * float64(dm.lockRetryMinInterval)))
+			// If we have been waiting for more than the force unlock timeout, return
+			// Remotes will have canceled due to the missing refreshes anyway.
+			if time.Since(started) > dm.clnt.Timeouts.UnlockCall {
+				return
+			}
+		}
+	}()
+}
+
+// sendRelease sends a release message to a node that previously granted a lock
+func sendRelease(ctx context.Context, ds *Dsync, c NetLocker, owner string, uid string, isReadLock bool, names ...string) bool {
+	if c == nil {
+		log("Unable to call RUnlock failed with %s\n", errors.New("netLocker is offline"))
+		return false
+	}
+
+	if len(uid) == 0 {
+		return false
+	}
+
+	args := LockArgs{
+		Owner:     owner,
+		UID:       uid,
+		Resources: names,
+	}
+
+	netLockCtx, cancel := context.WithTimeout(context.Background(), ds.Timeouts.UnlockCall)
+	defer cancel()
+
+	tc, ok := ctx.Value(mcontext.ContextTraceKey).(*mcontext.TraceCtxt)
+	if ok {
+		netLockCtx = context.WithValue(netLockCtx, mcontext.ContextTraceKey, tc)
+	}
+
+	if isReadLock {
+		if _, err := c.RUnlock(netLockCtx, args); err != nil {
+			log("dsync: Unable to call RUnlock failed with %s for %#v at %s\n", err, args, c)
+			return false
+		}
+	} else {
+		if _, err := c.Unlock(netLockCtx, args); err != nil {
+			log("dsync: Unable to call Unlock failed with %s for %#v at %s\n", err, args, c)
+			return false
+		}
+	}
+
+	return true
+}

internal/dsync/drwmutex_test.go

@@ -0,0 +1,355 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package dsync
+
+import (
+	"context"
+	"fmt"
+	"runtime"
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+const (
+	id     = "1234-5678"
+	source = "main.go"
+)
+
+func testSimpleWriteLock(t *testing.T, duration time.Duration) (locked bool) {
+	drwm1 := NewDRWMutex(ds, "simplelock")
+	ctx1, cancel1 := context.WithCancel(t.Context())
+	if !drwm1.GetRLock(ctx1, cancel1, id, source, Options{Timeout: time.Second}) {
+		panic("Failed to acquire read lock")
+	}
+	// fmt.Println("1st read lock acquired, waiting...")
+
+	drwm2 := NewDRWMutex(ds, "simplelock")
+	ctx2, cancel2 := context.WithCancel(t.Context())
+	if !drwm2.GetRLock(ctx2, cancel2, id, source, Options{Timeout: time.Second}) {
+		panic("Failed to acquire read lock")
+	}
+	// fmt.Println("2nd read lock acquired, waiting...")
+
+	go func() {
+		time.Sleep(2 * testDrwMutexAcquireTimeout)
+		drwm1.RUnlock(t.Context())
+		// fmt.Println("1st read lock released, waiting...")
+	}()
+
+	go func() {
+		time.Sleep(3 * testDrwMutexAcquireTimeout)
+		drwm2.RUnlock(t.Context())
+		// fmt.Println("2nd read lock released, waiting...")
+	}()
+
+	drwm3 := NewDRWMutex(ds, "simplelock")
+	// fmt.Println("Trying to acquire write lock, waiting...")
+	ctx3, cancel3 := context.WithCancel(t.Context())
+	locked = drwm3.GetLock(ctx3, cancel3, id, source, Options{Timeout: duration})
+	if locked {
+		// fmt.Println("Write lock acquired, waiting...")
+		time.Sleep(testDrwMutexAcquireTimeout)
+
+		drwm3.Unlock(t.Context())
+	}
+	// fmt.Println("Write lock failed due to timeout")
+	return locked
+}
+
+func TestSimpleWriteLockAcquired(t *testing.T) {
+	locked := testSimpleWriteLock(t, 10*testDrwMutexAcquireTimeout)
+
+	expected := true
+	if locked != expected {
+		t.Errorf("TestSimpleWriteLockAcquired(): \nexpected %#v\ngot      %#v", expected, locked)
+	}
+}
+
+func TestSimpleWriteLockTimedOut(t *testing.T) {
+	locked := testSimpleWriteLock(t, testDrwMutexAcquireTimeout)
+
+	expected := false
+	if locked != expected {
+		t.Errorf("TestSimpleWriteLockTimedOut(): \nexpected %#v\ngot      %#v", expected, locked)
+	}
+}
+
+func testDualWriteLock(t *testing.T, duration time.Duration) (locked bool) {
+	drwm1 := NewDRWMutex(ds, "duallock")
+
+	// fmt.Println("Getting initial write lock")
+	ctx1, cancel1 := context.WithCancel(t.Context())
+	if !drwm1.GetLock(ctx1, cancel1, id, source, Options{Timeout: time.Second}) {
+		panic("Failed to acquire initial write lock")
+	}
+
+	go func() {
+		time.Sleep(3 * testDrwMutexAcquireTimeout)
+		drwm1.Unlock(t.Context())
+		// fmt.Println("Initial write lock released, waiting...")
+	}()
+
+	// fmt.Println("Trying to acquire 2nd write lock, waiting...")
+	drwm2 := NewDRWMutex(ds, "duallock")
+	ctx2, cancel2 := context.WithCancel(t.Context())
+	locked = drwm2.GetLock(ctx2, cancel2, id, source, Options{Timeout: duration})
+	if locked {
+		// fmt.Println("2nd write lock acquired, waiting...")
+		time.Sleep(testDrwMutexAcquireTimeout)
+
+		drwm2.Unlock(t.Context())
+	}
+	// fmt.Println("2nd write lock failed due to timeout")
+	return locked
+}
+
+func TestDualWriteLockAcquired(t *testing.T) {
+	locked := testDualWriteLock(t, 10*testDrwMutexAcquireTimeout)
+
+	expected := true
+	if locked != expected {
+		t.Errorf("TestDualWriteLockAcquired(): \nexpected %#v\ngot      %#v", expected, locked)
+	}
+}
+
+func TestDualWriteLockTimedOut(t *testing.T) {
+	locked := testDualWriteLock(t, testDrwMutexAcquireTimeout)
+
+	expected := false
+	if locked != expected {
+		t.Errorf("TestDualWriteLockTimedOut(): \nexpected %#v\ngot      %#v", expected, locked)
+	}
+}
+
+// Test cases below are copied 1 to 1 from sync/rwmutex_test.go (adapted to use DRWMutex)
+
+// Borrowed from rwmutex_test.go
+func parallelReader(ctx context.Context, m *DRWMutex, clocked, cunlock, cdone chan bool) {
+	if m.GetRLock(ctx, nil, id, source, Options{Timeout: time.Second}) {
+		clocked <- true
+		<-cunlock
+		m.RUnlock(context.Background())
+		cdone <- true
+	}
+}
+
+// Borrowed from rwmutex_test.go
+func doTestParallelReaders(numReaders, gomaxprocs int) {
+	runtime.GOMAXPROCS(gomaxprocs)
+	m := NewDRWMutex(ds, "test-parallel")
+
+	clocked := make(chan bool)
+	cunlock := make(chan bool)
+	cdone := make(chan bool)
+	for range numReaders {
+		go parallelReader(context.Background(), m, clocked, cunlock, cdone)
+	}
+	// Wait for all parallel RLock()s to succeed.
+	for range numReaders {
+		<-clocked
+	}
+	for range numReaders {
+		cunlock <- true
+	}
+	// Wait for the goroutines to finish.
+	for range numReaders {
+		<-cdone
+	}
+}
+
+// Borrowed from rwmutex_test.go
+func TestParallelReaders(t *testing.T) {
+	defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(-1))
+	doTestParallelReaders(1, 4)
+	doTestParallelReaders(3, 4)
+	doTestParallelReaders(4, 2)
+}
+
+// Borrowed from rwmutex_test.go
+func reader(resource string, numIterations int, activity *int32, cdone chan bool) {
+	rwm := NewDRWMutex(ds, resource)
+	for range numIterations {
+		if rwm.GetRLock(context.Background(), nil, id, source, Options{Timeout: time.Second}) {
+			n := atomic.AddInt32(activity, 1)
+			if n < 1 || n >= 10000 {
+				panic(fmt.Sprintf("wlock(%d)\n", n))
+			}
+			for range 100 {
+			}
+			atomic.AddInt32(activity, -1)
+			rwm.RUnlock(context.Background())
+		}
+	}
+	cdone <- true
+}
+
+// Borrowed from rwmutex_test.go
+func writer(resource string, numIterations int, activity *int32, cdone chan bool) {
+	rwm := NewDRWMutex(ds, resource)
+	for range numIterations {
+		if rwm.GetLock(context.Background(), nil, id, source, Options{Timeout: time.Second}) {
+			n := atomic.AddInt32(activity, 10000)
+			if n != 10000 {
+				panic(fmt.Sprintf("wlock(%d)\n", n))
+			}
+			for range 100 {
+			}
+			atomic.AddInt32(activity, -10000)
+			rwm.Unlock(context.Background())
+		}
+	}
+	cdone <- true
+}
+
+// Borrowed from rwmutex_test.go
+func hammerRWMutex(t *testing.T, gomaxprocs, numReaders, numIterations int) {
+	t.Run(fmt.Sprintf("%d-%d-%d", gomaxprocs, numReaders, numIterations), func(t *testing.T) {
+		resource := "test"
+		runtime.GOMAXPROCS(gomaxprocs)
+		// Number of active readers + 10000 * number of active writers.
+		var activity int32
+		cdone := make(chan bool)
+		go writer(resource, numIterations, &activity, cdone)
+		var i int
+		for i = 0; i < numReaders/2; i++ {
+			go reader(resource, numIterations, &activity, cdone)
+		}
+		go writer(resource, numIterations, &activity, cdone)
+		for ; i < numReaders; i++ {
+			go reader(resource, numIterations, &activity, cdone)
+		}
+		// Wait for the 2 writers and all readers to finish.
+		for i := 0; i < 2+numReaders; i++ {
+			<-cdone
+		}
+	})
+}
+
+// Borrowed from rwmutex_test.go
+func TestRWMutex(t *testing.T) {
+	defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(-1))
+	n := 100
+	if testing.Short() {
+		n = 5
+	}
+	hammerRWMutex(t, 1, 1, n)
+	hammerRWMutex(t, 1, 3, n)
+	hammerRWMutex(t, 1, 10, n)
+	hammerRWMutex(t, 4, 1, n)
+	hammerRWMutex(t, 4, 3, n)
+	hammerRWMutex(t, 4, 10, n)
+	hammerRWMutex(t, 10, 1, n)
+	hammerRWMutex(t, 10, 3, n)
+	hammerRWMutex(t, 10, 10, n)
+	hammerRWMutex(t, 10, 5, n)
+}
+
+// Borrowed from rwmutex_test.go
+func TestUnlockPanic(t *testing.T) {
+	defer func() {
+		if recover() == nil {
+			t.Fatalf("unlock of unlocked RWMutex did not panic")
+		}
+	}()
+	mu := NewDRWMutex(ds, "test")
+	mu.Unlock(t.Context())
+}
+
+// Borrowed from rwmutex_test.go
+func TestUnlockPanic2(t *testing.T) {
+	mu := NewDRWMutex(ds, "test-unlock-panic-2")
+	defer func() {
+		if recover() == nil {
+			t.Fatalf("unlock of unlocked RWMutex did not panic")
+		}
+		mu.RUnlock(t.Context()) // Unlock, so -test.count > 1 works
+	}()
+	mu.RLock(id, source)
+	mu.Unlock(t.Context())
+}
+
+// Borrowed from rwmutex_test.go
+func TestRUnlockPanic(t *testing.T) {
+	defer func() {
+		if recover() == nil {
+			t.Fatalf("read unlock of unlocked RWMutex did not panic")
+		}
+	}()
+	mu := NewDRWMutex(ds, "test")
+	mu.RUnlock(t.Context())
+}
+
+// Borrowed from rwmutex_test.go
+func TestRUnlockPanic2(t *testing.T) {
+	mu := NewDRWMutex(ds, "test-runlock-panic-2")
+	defer func() {
+		if recover() == nil {
+			t.Fatalf("read unlock of unlocked RWMutex did not panic")
+		}
+		mu.Unlock(t.Context()) // Unlock, so -test.count > 1 works
+	}()
+	mu.Lock(id, source)
+	mu.RUnlock(t.Context())
+}
+
+// Borrowed from rwmutex_test.go
+func benchmarkRWMutex(b *testing.B, localWork, writeRatio int) {
+	b.ResetTimer()
+	b.ReportAllocs()
+
+	b.RunParallel(func(pb *testing.PB) {
+		foo := 0
+		for pb.Next() {
+			rwm := NewDRWMutex(ds, "test")
+			foo++
+			if foo%writeRatio == 0 {
+				rwm.Lock(id, source)
+				rwm.Unlock(b.Context())
+			} else {
+				rwm.RLock(id, source)
+				for i := 0; i != localWork; i++ {
+					foo *= 2
+					foo /= 2
+				}
+				rwm.RUnlock(b.Context())
+			}
+		}
+		_ = foo
+	})
+}
+
+// Borrowed from rwmutex_test.go
+func BenchmarkRWMutexWrite100(b *testing.B) {
+	benchmarkRWMutex(b, 0, 100)
+}
+
+// Borrowed from rwmutex_test.go
+func BenchmarkRWMutexWrite10(b *testing.B) {
+	benchmarkRWMutex(b, 0, 10)
+}
+
+// Borrowed from rwmutex_test.go
+func BenchmarkRWMutexWorkWrite100(b *testing.B) {
+	benchmarkRWMutex(b, 100, 100)
+}
+
+// Borrowed from rwmutex_test.go
+func BenchmarkRWMutexWorkWrite10(b *testing.B) {
+	benchmarkRWMutex(b, 100, 10)
+}

internal/dsync/dsync-client_test.go

@@ -0,0 +1,149 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package dsync
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"net/http"
+	"net/url"
+	"time"
+
+	xhttp "github.com/minio/minio/internal/http"
+	"github.com/minio/minio/internal/rest"
+)
+
+// ReconnectRESTClient is a wrapper type for rest.Client which provides reconnect on first failure.
+type ReconnectRESTClient struct {
+	u    *url.URL
+	rest *rest.Client
+}
+
+// newClient constructs a ReconnectRESTClient object with addr and endpoint initialized.
+// It _doesn't_ connect to the remote endpoint. See Call method to see when the
+// connect happens.
+func newClient(endpoint string) NetLocker {
+	u, err := url.Parse(endpoint)
+	if err != nil {
+		panic(err)
+	}
+
+	tr := &http.Transport{
+		Proxy:                 http.ProxyFromEnvironment,
+		MaxIdleConnsPerHost:   1024,
+		WriteBufferSize:       32 << 10, // 32KiB moving up from 4KiB default
+		ReadBufferSize:        32 << 10, // 32KiB moving up from 4KiB default
+		IdleConnTimeout:       15 * time.Second,
+		ResponseHeaderTimeout: 15 * time.Minute, // Set conservative timeouts for MinIO internode.
+		TLSHandshakeTimeout:   15 * time.Second,
+		ExpectContinueTimeout: 15 * time.Second,
+		// Go net/http automatically unzip if content-type is
+		// gzip disable this feature, as we are always interested
+		// in raw stream.
+		DisableCompression: true,
+	}
+
+	return &ReconnectRESTClient{
+		u:    u,
+		rest: rest.NewClient(u, tr, nil),
+	}
+}
+
+// Close closes the underlying socket file descriptor.
+func (restClient *ReconnectRESTClient) IsOnline() bool {
+	// If rest client has not connected yet there is nothing to close.
+	return restClient.rest != nil
+}
+
+func (restClient *ReconnectRESTClient) IsLocal() bool {
+	return false
+}
+
+// Close closes the underlying socket file descriptor.
+func (restClient *ReconnectRESTClient) Close() error {
+	return nil
+}
+
+var (
+	errLockConflict = errors.New("lock conflict")
+	errLockNotFound = errors.New("lock not found")
+)
+
+func toLockError(err error) error {
+	if err == nil {
+		return nil
+	}
+
+	switch err.Error() {
+	case errLockConflict.Error():
+		return errLockConflict
+	case errLockNotFound.Error():
+		return errLockNotFound
+	}
+	return err
+}
+
+// Call makes a REST call to the remote endpoint using the msgp codec
+func (restClient *ReconnectRESTClient) Call(method string, args LockArgs) (status bool, err error) {
+	buf, err := args.MarshalMsg(nil)
+	if err != nil {
+		return false, err
+	}
+	body := bytes.NewReader(buf)
+	respBody, err := restClient.rest.Call(context.Background(), method,
+		url.Values{}, body, body.Size())
+	defer xhttp.DrainBody(respBody)
+
+	switch toLockError(err) {
+	case nil:
+		return true, nil
+	case errLockConflict, errLockNotFound:
+		return false, nil
+	default:
+		return false, err
+	}
+}
+
+func (restClient *ReconnectRESTClient) RLock(ctx context.Context, args LockArgs) (status bool, err error) {
+	return restClient.Call("/v1/rlock", args)
+}
+
+func (restClient *ReconnectRESTClient) Lock(ctx context.Context, args LockArgs) (status bool, err error) {
+	return restClient.Call("/v1/lock", args)
+}
+
+func (restClient *ReconnectRESTClient) RUnlock(ctx context.Context, args LockArgs) (status bool, err error) {
+	return restClient.Call("/v1/runlock", args)
+}
+
+func (restClient *ReconnectRESTClient) Unlock(ctx context.Context, args LockArgs) (status bool, err error) {
+	return restClient.Call("/v1/unlock", args)
+}
+
+func (restClient *ReconnectRESTClient) Refresh(ctx context.Context, args LockArgs) (refreshed bool, err error) {
+	return restClient.Call("/v1/refresh", args)
+}
+
+func (restClient *ReconnectRESTClient) ForceUnlock(ctx context.Context, args LockArgs) (reply bool, err error) {
+	return restClient.Call("/v1/force-unlock", args)
+}
+
+func (restClient *ReconnectRESTClient) String() string {
+	return restClient.u.String()
+}

internal/dsync/dsync-server_test.go

@@ -0,0 +1,304 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package dsync
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/minio/mux"
+)
+
+const numberOfNodes = 5
+
+var (
+	ds          *Dsync
+	nodes       = make([]*httptest.Server, numberOfNodes) // list of node IP addrs or hostname with ports.
+	lockServers = make([]*lockServer, numberOfNodes)
+)
+
+func getLockArgs(r *http.Request) (args LockArgs, err error) {
+	buf, err := io.ReadAll(r.Body)
+	if err != nil {
+		return args, err
+	}
+	_, err = args.UnmarshalMsg(buf)
+	return args, err
+}
+
+type lockServerHandler struct {
+	lsrv *lockServer
+}
+
+func (lh *lockServerHandler) writeErrorResponse(w http.ResponseWriter, err error) {
+	w.WriteHeader(http.StatusForbidden)
+	w.Write([]byte(err.Error()))
+}
+
+func (lh *lockServerHandler) ForceUnlockHandler(w http.ResponseWriter, r *http.Request) {
+	args, err := getLockArgs(r)
+	if err != nil {
+		lh.writeErrorResponse(w, err)
+		return
+	}
+
+	if _, err = lh.lsrv.ForceUnlock(&args); err != nil {
+		lh.writeErrorResponse(w, err)
+		return
+	}
+}
+
+func (lh *lockServerHandler) RefreshHandler(w http.ResponseWriter, r *http.Request) {
+	args, err := getLockArgs(r)
+	if err != nil {
+		lh.writeErrorResponse(w, err)
+		return
+	}
+
+	reply, err := lh.lsrv.Refresh(&args)
+	if err != nil {
+		lh.writeErrorResponse(w, err)
+		return
+	}
+
+	if !reply {
+		lh.writeErrorResponse(w, errLockNotFound)
+		return
+	}
+}
+
+func (lh *lockServerHandler) LockHandler(w http.ResponseWriter, r *http.Request) {
+	args, err := getLockArgs(r)
+	if err != nil {
+		lh.writeErrorResponse(w, err)
+		return
+	}
+	reply, err := lh.lsrv.Lock(&args)
+	if err == nil && !reply {
+		err = errLockConflict
+	}
+	if err != nil {
+		lh.writeErrorResponse(w, err)
+		return
+	}
+}
+
+func (lh *lockServerHandler) UnlockHandler(w http.ResponseWriter, r *http.Request) {
+	args, err := getLockArgs(r)
+	if err != nil {
+		lh.writeErrorResponse(w, err)
+		return
+	}
+	_, err = lh.lsrv.Unlock(&args)
+	if err != nil {
+		lh.writeErrorResponse(w, err)
+		return
+	}
+}
+
+func (lh *lockServerHandler) RUnlockHandler(w http.ResponseWriter, r *http.Request) {
+	args, err := getLockArgs(r)
+	if err != nil {
+		lh.writeErrorResponse(w, err)
+		return
+	}
+	_, err = lh.lsrv.RUnlock(&args)
+	if err != nil {
+		lh.writeErrorResponse(w, err)
+		return
+	}
+}
+
+func (lh *lockServerHandler) HealthHandler(w http.ResponseWriter, r *http.Request) {}
+
+func (lh *lockServerHandler) RLockHandler(w http.ResponseWriter, r *http.Request) {
+	args, err := getLockArgs(r)
+	if err != nil {
+		lh.writeErrorResponse(w, err)
+		return
+	}
+
+	reply, err := lh.lsrv.RLock(&args)
+	if err == nil && !reply {
+		err = errLockConflict
+	}
+	if err != nil {
+		lh.writeErrorResponse(w, err)
+		return
+	}
+}
+
+func stopLockServers() {
+	for i := range numberOfNodes {
+		nodes[i].Close()
+	}
+}
+
+func startLockServers() {
+	for i := range numberOfNodes {
+		lsrv := &lockServer{
+			mutex:   sync.Mutex{},
+			lockMap: make(map[string]int64),
+		}
+		lockServer := lockServerHandler{
+			lsrv: lsrv,
+		}
+		lockServers[i] = lsrv
+
+		router := mux.NewRouter().SkipClean(true)
+		subrouter := router.PathPrefix("/").Subrouter()
+		subrouter.Methods(http.MethodPost).Path("/v1/health").HandlerFunc(lockServer.HealthHandler)
+		subrouter.Methods(http.MethodPost).Path("/v1/refresh").HandlerFunc(lockServer.RefreshHandler)
+		subrouter.Methods(http.MethodPost).Path("/v1/lock").HandlerFunc(lockServer.LockHandler)
+		subrouter.Methods(http.MethodPost).Path("/v1/rlock").HandlerFunc(lockServer.RLockHandler)
+		subrouter.Methods(http.MethodPost).Path("/v1/unlock").HandlerFunc(lockServer.UnlockHandler)
+		subrouter.Methods(http.MethodPost).Path("/v1/runlock").HandlerFunc(lockServer.RUnlockHandler)
+		subrouter.Methods(http.MethodPost).Path("/v1/force-unlock").HandlerFunc(lockServer.ForceUnlockHandler)
+
+		nodes[i] = httptest.NewServer(router)
+	}
+}
+
+const WriteLock = -1
+
+type lockServer struct {
+	mutex sync.Mutex
+	// Map of locks, with negative value indicating (exclusive) write lock
+	// and positive values indicating number of read locks
+	lockMap map[string]int64
+
+	// Refresh returns lock not found if set to true
+	lockNotFound bool
+
+	// Set to true if you want peers servers to do not respond
+	responseDelay int64
+}
+
+func (l *lockServer) setRefreshReply(refreshed bool) {
+	l.mutex.Lock()
+	defer l.mutex.Unlock()
+	l.lockNotFound = !refreshed
+}
+
+func (l *lockServer) setResponseDelay(responseDelay time.Duration) {
+	atomic.StoreInt64(&l.responseDelay, int64(responseDelay))
+}
+
+func (l *lockServer) Lock(args *LockArgs) (reply bool, err error) {
+	if d := atomic.LoadInt64(&l.responseDelay); d != 0 {
+		time.Sleep(time.Duration(d))
+	}
+
+	l.mutex.Lock()
+	defer l.mutex.Unlock()
+	if _, reply = l.lockMap[args.Resources[0]]; !reply {
+		l.lockMap[args.Resources[0]] = WriteLock // No locks held on the given name, so claim write lock
+	}
+	reply = !reply // Negate *reply to return true when lock is granted or false otherwise
+	return reply, nil
+}
+
+func (l *lockServer) Unlock(args *LockArgs) (reply bool, err error) {
+	if d := atomic.LoadInt64(&l.responseDelay); d != 0 {
+		time.Sleep(time.Duration(d))
+	}
+
+	l.mutex.Lock()
+	defer l.mutex.Unlock()
+	var locksHeld int64
+	if locksHeld, reply = l.lockMap[args.Resources[0]]; !reply { // No lock is held on the given name
+		return false, fmt.Errorf("Unlock attempted on an unlocked entity: %s", args.Resources[0])
+	}
+	if reply = locksHeld == WriteLock; !reply { // Unless it is a write lock
+		return false, fmt.Errorf("Unlock attempted on a read locked entity: %s (%d read locks active)", args.Resources[0], locksHeld)
+	}
+	delete(l.lockMap, args.Resources[0]) // Remove the write lock
+	return true, nil
+}
+
+const ReadLock = 1
+
+func (l *lockServer) RLock(args *LockArgs) (reply bool, err error) {
+	if d := atomic.LoadInt64(&l.responseDelay); d != 0 {
+		time.Sleep(time.Duration(d))
+	}
+
+	l.mutex.Lock()
+	defer l.mutex.Unlock()
+	var locksHeld int64
+	if locksHeld, reply = l.lockMap[args.Resources[0]]; !reply {
+		l.lockMap[args.Resources[0]] = ReadLock // No locks held on the given name, so claim (first) read lock
+		reply = true
+	} else if reply = locksHeld != WriteLock; reply { // Unless there is a write lock
+		l.lockMap[args.Resources[0]] = locksHeld + ReadLock // Grant another read lock
+	}
+	return reply, nil
+}
+
+func (l *lockServer) RUnlock(args *LockArgs) (reply bool, err error) {
+	if d := atomic.LoadInt64(&l.responseDelay); d != 0 {
+		time.Sleep(time.Duration(d))
+	}
+
+	l.mutex.Lock()
+	defer l.mutex.Unlock()
+	var locksHeld int64
+	if locksHeld, reply = l.lockMap[args.Resources[0]]; !reply { // No lock is held on the given name
+		return false, fmt.Errorf("RUnlock attempted on an unlocked entity: %s", args.Resources[0])
+	}
+	if reply = locksHeld != WriteLock; !reply { // A write-lock is held, cannot release a read lock
+		return false, fmt.Errorf("RUnlock attempted on a write locked entity: %s", args.Resources[0])
+	}
+	if locksHeld > ReadLock {
+		l.lockMap[args.Resources[0]] = locksHeld - ReadLock // Remove one of the read locks held
+	} else {
+		delete(l.lockMap, args.Resources[0]) // Remove the (last) read lock
+	}
+	return reply, nil
+}
+
+func (l *lockServer) Refresh(args *LockArgs) (reply bool, err error) {
+	if d := atomic.LoadInt64(&l.responseDelay); d != 0 {
+		time.Sleep(time.Duration(d))
+	}
+
+	l.mutex.Lock()
+	defer l.mutex.Unlock()
+	reply = !l.lockNotFound
+	return reply, nil
+}
+
+func (l *lockServer) ForceUnlock(args *LockArgs) (reply bool, err error) {
+	if d := atomic.LoadInt64(&l.responseDelay); d != 0 {
+		time.Sleep(time.Duration(d))
+	}
+
+	l.mutex.Lock()
+	defer l.mutex.Unlock()
+	if len(args.UID) != 0 {
+		return false, fmt.Errorf("ForceUnlock called with non-empty UID: %s", args.UID)
+	}
+	delete(l.lockMap, args.Resources[0]) // Remove the lock (irrespective of write or read lock)
+	reply = true
+	return reply, nil
+}

internal/dsync/dsync.go

@@ -0,0 +1,28 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package dsync
+
+// Dsync represents dsync client object which is initialized with
+// authenticated clients, used to initiate lock REST calls.
+type Dsync struct {
+	// List of rest client objects, one per lock server.
+	GetLockers func() ([]NetLocker, string)
+
+	// Timeouts to apply.
+	Timeouts Timeouts
+}

internal/dsync/dsync_test.go

@@ -0,0 +1,451 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package dsync
+
+import (
+	"context"
+	"math/rand"
+	"os"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/google/uuid"
+)
+
+const (
+	testDrwMutexAcquireTimeout         = 250 * time.Millisecond
+	testDrwMutexRefreshCallTimeout     = 250 * time.Millisecond
+	testDrwMutexUnlockCallTimeout      = 250 * time.Millisecond
+	testDrwMutexForceUnlockCallTimeout = 250 * time.Millisecond
+	testDrwMutexRefreshInterval        = 100 * time.Millisecond
+)
+
+// TestMain initializes the testing framework
+func TestMain(m *testing.M) {
+	startLockServers()
+
+	// Initialize locker clients for dsync.
+	var clnts []NetLocker
+	for i := range nodes {
+		clnts = append(clnts, newClient(nodes[i].URL))
+	}
+
+	ds = &Dsync{
+		GetLockers: func() ([]NetLocker, string) { return clnts, uuid.New().String() },
+		Timeouts: Timeouts{
+			Acquire:         testDrwMutexAcquireTimeout,
+			RefreshCall:     testDrwMutexRefreshCallTimeout,
+			UnlockCall:      testDrwMutexUnlockCallTimeout,
+			ForceUnlockCall: testDrwMutexForceUnlockCallTimeout,
+		},
+	}
+
+	code := m.Run()
+	stopLockServers()
+	os.Exit(code)
+}
+
+func TestSimpleLock(t *testing.T) {
+	dm := NewDRWMutex(ds, "test")
+
+	dm.Lock(id, source)
+
+	// fmt.Println("Lock acquired, waiting...")
+	time.Sleep(testDrwMutexRefreshCallTimeout)
+
+	dm.Unlock(t.Context())
+}
+
+func TestSimpleLockUnlockMultipleTimes(t *testing.T) {
+	dm := NewDRWMutex(ds, "test")
+
+	dm.Lock(id, source)
+	time.Sleep(time.Duration(10+(rand.Float32()*50)) * time.Millisecond)
+	dm.Unlock(t.Context())
+
+	dm.Lock(id, source)
+	time.Sleep(time.Duration(10+(rand.Float32()*50)) * time.Millisecond)
+	dm.Unlock(t.Context())
+
+	dm.Lock(id, source)
+	time.Sleep(time.Duration(10+(rand.Float32()*50)) * time.Millisecond)
+	dm.Unlock(t.Context())
+
+	dm.Lock(id, source)
+	time.Sleep(time.Duration(10+(rand.Float32()*50)) * time.Millisecond)
+	dm.Unlock(t.Context())
+
+	dm.Lock(id, source)
+	time.Sleep(time.Duration(10+(rand.Float32()*50)) * time.Millisecond)
+	dm.Unlock(t.Context())
+}
+
+// Test two locks for same resource, one succeeds, one fails (after timeout)
+func TestTwoSimultaneousLocksForSameResource(t *testing.T) {
+	dm1st := NewDRWMutex(ds, "aap")
+	dm2nd := NewDRWMutex(ds, "aap")
+
+	dm1st.Lock(id, source)
+
+	// Release lock after 10 seconds
+	go func() {
+		time.Sleep(5 * testDrwMutexAcquireTimeout)
+		// fmt.Println("Unlocking dm1")
+
+		dm1st.Unlock(t.Context())
+	}()
+
+	dm2nd.Lock(id, source)
+
+	// fmt.Printf("2nd lock obtained after 1st lock is released\n")
+	time.Sleep(testDrwMutexRefreshCallTimeout * 2)
+
+	dm2nd.Unlock(t.Context())
+}
+
+// Test three locks for same resource, one succeeds, one fails (after timeout)
+func TestThreeSimultaneousLocksForSameResource(t *testing.T) {
+	dm1st := NewDRWMutex(ds, "aap")
+	dm2nd := NewDRWMutex(ds, "aap")
+	dm3rd := NewDRWMutex(ds, "aap")
+
+	dm1st.Lock(id, source)
+	started := time.Now()
+	var expect time.Duration
+	// Release lock after 10 seconds
+	go func() {
+		// TOTAL
+		time.Sleep(2 * testDrwMutexAcquireTimeout)
+		// fmt.Println("Unlocking dm1")
+
+		dm1st.Unlock(t.Context())
+	}()
+	expect += 2 * testDrwMutexAcquireTimeout
+
+	var wg sync.WaitGroup
+	wg.Add(2)
+
+	go func() {
+		defer wg.Done()
+
+		dm2nd.Lock(id, source)
+
+		// Release lock after 10 seconds
+		go func() {
+			time.Sleep(2 * testDrwMutexAcquireTimeout)
+			// fmt.Println("Unlocking dm2")
+
+			dm2nd.Unlock(t.Context())
+		}()
+
+		dm3rd.Lock(id, source)
+
+		// fmt.Printf("3rd lock obtained after 1st & 2nd locks are released\n")
+		time.Sleep(testDrwMutexRefreshCallTimeout)
+
+		dm3rd.Unlock(t.Context())
+	}()
+	expect += 2*testDrwMutexAcquireTimeout + testDrwMutexRefreshCallTimeout
+
+	go func() {
+		defer wg.Done()
+
+		dm3rd.Lock(id, source)
+
+		// Release lock after 10 seconds
+		go func() {
+			time.Sleep(2 * testDrwMutexAcquireTimeout)
+			// fmt.Println("Unlocking dm3")
+
+			dm3rd.Unlock(t.Context())
+		}()
+
+		dm2nd.Lock(id, source)
+
+		// fmt.Printf("2nd lock obtained after 1st & 3rd locks are released\n")
+		time.Sleep(testDrwMutexRefreshCallTimeout)
+
+		dm2nd.Unlock(t.Context())
+	}()
+	expect += 2*testDrwMutexAcquireTimeout + testDrwMutexRefreshCallTimeout
+
+	wg.Wait()
+	// We expect at least 3 x 2 x testDrwMutexAcquireTimeout to have passed
+	elapsed := time.Since(started)
+	if elapsed < expect {
+		t.Errorf("expected at least %v time have passed, however %v passed", expect, elapsed)
+	}
+	t.Logf("expected at least %v time have passed, %v passed", expect, elapsed)
+}
+
+// Test two locks for different resources, both succeed
+func TestTwoSimultaneousLocksForDifferentResources(t *testing.T) {
+	dm1 := NewDRWMutex(ds, "aap")
+	dm2 := NewDRWMutex(ds, "noot")
+
+	dm1.Lock(id, source)
+	dm2.Lock(id, source)
+	dm1.Unlock(t.Context())
+	dm2.Unlock(t.Context())
+}
+
+// Test refreshing lock - refresh should always return true
+func TestSuccessfulLockRefresh(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping test in short mode.")
+	}
+
+	dm := NewDRWMutex(ds, "aap")
+	dm.refreshInterval = testDrwMutexRefreshInterval
+
+	ctx, cancel := context.WithCancel(t.Context())
+
+	if !dm.GetLock(ctx, cancel, id, source, Options{Timeout: 5 * time.Minute}) {
+		t.Fatal("GetLock() should be successful")
+	}
+
+	// Make it run twice.
+	timer := time.NewTimer(testDrwMutexRefreshInterval * 2)
+
+	select {
+	case <-ctx.Done():
+		t.Fatal("Lock context canceled which is not expected")
+	case <-timer.C:
+	}
+
+	// Should be safe operation in all cases
+	dm.Unlock(t.Context())
+}
+
+// Test canceling context while quorum servers report lock not found
+func TestFailedRefreshLock(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping test in short mode.")
+	}
+
+	// Simulate Refresh response to return no locking found
+	for i := range lockServers[:3] {
+		lockServers[i].setRefreshReply(false)
+		defer lockServers[i].setRefreshReply(true)
+	}
+
+	dm := NewDRWMutex(ds, "aap")
+	dm.refreshInterval = 500 * time.Millisecond
+	var wg sync.WaitGroup
+	wg.Add(1)
+
+	ctx, cl := context.WithCancel(t.Context())
+	cancel := func() {
+		cl()
+		wg.Done()
+	}
+
+	if !dm.GetLock(ctx, cancel, id, source, Options{Timeout: 5 * time.Minute}) {
+		t.Fatal("GetLock() should be successful")
+	}
+
+	// Wait until context is canceled
+	wg.Wait()
+	if ctx.Err() == nil {
+		t.Fatal("Unexpected error", ctx.Err())
+	}
+
+	// Should be safe operation in all cases
+	dm.Unlock(t.Context())
+}
+
+// Test Unlock should not timeout
+func TestUnlockShouldNotTimeout(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping test in short mode.")
+	}
+
+	dm := NewDRWMutex(ds, "aap")
+	dm.refreshInterval = testDrwMutexUnlockCallTimeout
+	if !dm.GetLock(t.Context(), nil, id, source, Options{Timeout: 5 * time.Minute}) {
+		t.Fatal("GetLock() should be successful")
+	}
+
+	// Add delay to lock server responses to ensure that lock does not timeout
+	for i := range lockServers {
+		lockServers[i].setResponseDelay(5 * testDrwMutexUnlockCallTimeout)
+		defer lockServers[i].setResponseDelay(0)
+	}
+
+	unlockReturned := make(chan struct{}, 1)
+	go func() {
+		ctx, cancel := context.WithTimeout(t.Context(), 500*time.Millisecond)
+		defer cancel()
+		dm.Unlock(ctx)
+		// Unlock is not blocking. Try to get a new lock.
+		dm.GetLock(ctx, nil, id, source, Options{Timeout: 5 * time.Minute})
+		unlockReturned <- struct{}{}
+	}()
+
+	timer := time.NewTimer(2 * testDrwMutexUnlockCallTimeout)
+	defer timer.Stop()
+
+	select {
+	case <-unlockReturned:
+		t.Fatal("Unlock timed out, which should not happen")
+	case <-timer.C:
+	}
+}
+
+// Borrowed from mutex_test.go
+func HammerMutex(m *DRWMutex, loops int, cdone chan bool) {
+	for range loops {
+		m.Lock(id, source)
+		m.Unlock(context.Background())
+	}
+	cdone <- true
+}
+
+// Borrowed from mutex_test.go
+func TestMutex(t *testing.T) {
+	loops := 200
+	if testing.Short() {
+		loops = 5
+	}
+	c := make(chan bool)
+	m := NewDRWMutex(ds, "test")
+	for range 10 {
+		go HammerMutex(m, loops, c)
+	}
+	for range 10 {
+		<-c
+	}
+}
+
+func BenchmarkMutexUncontended(b *testing.B) {
+	b.ResetTimer()
+	b.ReportAllocs()
+
+	type PaddedMutex struct {
+		*DRWMutex
+	}
+	b.RunParallel(func(pb *testing.PB) {
+		mu := PaddedMutex{NewDRWMutex(ds, "")}
+		for pb.Next() {
+			mu.Lock(id, source)
+			mu.Unlock(b.Context())
+		}
+	})
+}
+
+func benchmarkMutex(b *testing.B, slack, work bool) {
+	b.ResetTimer()
+	b.ReportAllocs()
+
+	mu := NewDRWMutex(ds, "")
+	if slack {
+		b.SetParallelism(10)
+	}
+	b.RunParallel(func(pb *testing.PB) {
+		foo := 0
+		for pb.Next() {
+			mu.Lock(id, source)
+			mu.Unlock(b.Context())
+			if work {
+				for range 100 {
+					foo *= 2
+					foo /= 2
+				}
+			}
+		}
+		_ = foo
+	})
+}
+
+func BenchmarkMutex(b *testing.B) {
+	benchmarkMutex(b, false, false)
+}
+
+func BenchmarkMutexSlack(b *testing.B) {
+	benchmarkMutex(b, true, false)
+}
+
+func BenchmarkMutexWork(b *testing.B) {
+	benchmarkMutex(b, false, true)
+}
+
+func BenchmarkMutexWorkSlack(b *testing.B) {
+	benchmarkMutex(b, true, true)
+}
+
+func BenchmarkMutexNoSpin(b *testing.B) {
+	b.ResetTimer()
+	b.ReportAllocs()
+
+	// This benchmark models a situation where spinning in the mutex should be
+	// non-profitable and allows to confirm that spinning does not do harm.
+	// To achieve this we create excess of goroutines most of which do local work.
+	// These goroutines yield during local work, so that switching from
+	// a blocked goroutine to other goroutines is profitable.
+	// As a matter of fact, this benchmark still triggers some spinning in the mutex.
+	m := NewDRWMutex(ds, "")
+	var acc0, acc1 uint64
+	b.SetParallelism(4)
+	b.RunParallel(func(pb *testing.PB) {
+		c := make(chan bool)
+		var data [4 << 10]uint64
+		for i := 0; pb.Next(); i++ {
+			if i%4 == 0 {
+				m.Lock(id, source)
+				acc0 -= 100
+				acc1 += 100
+				m.Unlock(b.Context())
+			} else {
+				for i := 0; i < len(data); i += 4 {
+					data[i]++
+				}
+				// Elaborate way to say runtime.Gosched
+				// that does not put the goroutine onto global runq.
+				go func() {
+					c <- true
+				}()
+				<-c
+			}
+		}
+	})
+}
+
+func BenchmarkMutexSpin(b *testing.B) {
+	b.ResetTimer()
+	b.ReportAllocs()
+
+	// This benchmark models a situation where spinning in the mutex should be
+	// profitable. To achieve this we create a goroutine per-proc.
+	// These goroutines access considerable amount of local data so that
+	// unnecessary rescheduling is penalized by cache misses.
+	m := NewDRWMutex(ds, "")
+	var acc0, acc1 uint64
+	b.RunParallel(func(pb *testing.PB) {
+		var data [16 << 10]uint64
+		for i := 0; pb.Next(); i++ {
+			m.Lock(id, source)
+			acc0 -= 100
+			acc1 += 100
+			m.Unlock(b.Context())
+			for i := 0; i < len(data); i += 4 {
+				data[i]++
+			}
+		}
+	})
+}

internal/dsync/lock-args.go

@@ -0,0 +1,58 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package dsync
+
+//go:generate msgp -file $GOFILE
+
+// LockArgs is minimal required values for any dsync compatible lock operation.
+type LockArgs struct {
+	// Unique ID of lock/unlock request.
+	UID string
+
+	// Resources contains single or multiple entries to be locked/unlocked.
+	Resources []string
+
+	// Owner represents unique ID for this instance, an owner who originally requested
+	// the locked resource, useful primarily in figuring out stale locks.
+	Owner string
+
+	// Source contains the line number, function and file name of the code
+	// on the client node that requested the lock.
+	Source string `msgp:"omitempty"`
+
+	// Quorum represents the expected quorum for this lock type.
+	Quorum *int `msgp:"omitempty"`
+}
+
+// ResponseCode is the response code for a locking request.
+type ResponseCode uint8
+
+// Response codes for a locking request.
+const (
+	RespOK ResponseCode = iota
+	RespLockConflict
+	RespLockNotInitialized
+	RespLockNotFound
+	RespErr
+)
+
+// LockResp is a locking request response.
+type LockResp struct {
+	Code ResponseCode
+	Err  string
+}

internal/dsync/lock-args_gen.go

@@ -0,0 +1,477 @@
+// Code generated by github.com/tinylib/msgp DO NOT EDIT.
+
+package dsync
+
+import (
+	"github.com/tinylib/msgp/msgp"
+)
+
+// DecodeMsg implements msgp.Decodable
+func (z *LockArgs) DecodeMsg(dc *msgp.Reader) (err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, err = dc.ReadMapHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, err = dc.ReadMapKeyPtr()
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "UID":
+			z.UID, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "UID")
+				return
+			}
+		case "Resources":
+			var zb0002 uint32
+			zb0002, err = dc.ReadArrayHeader()
+			if err != nil {
+				err = msgp.WrapError(err, "Resources")
+				return
+			}
+			if cap(z.Resources) >= int(zb0002) {
+				z.Resources = (z.Resources)[:zb0002]
+			} else {
+				z.Resources = make([]string, zb0002)
+			}
+			for za0001 := range z.Resources {
+				z.Resources[za0001], err = dc.ReadString()
+				if err != nil {
+					err = msgp.WrapError(err, "Resources", za0001)
+					return
+				}
+			}
+		case "Owner":
+			z.Owner, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "Owner")
+				return
+			}
+		case "Source":
+			z.Source, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "Source")
+				return
+			}
+		case "Quorum":
+			if dc.IsNil() {
+				err = dc.ReadNil()
+				if err != nil {
+					err = msgp.WrapError(err, "Quorum")
+					return
+				}
+				z.Quorum = nil
+			} else {
+				if z.Quorum == nil {
+					z.Quorum = new(int)
+				}
+				*z.Quorum, err = dc.ReadInt()
+				if err != nil {
+					err = msgp.WrapError(err, "Quorum")
+					return
+				}
+			}
+		default:
+			err = dc.Skip()
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z *LockArgs) EncodeMsg(en *msgp.Writer) (err error) {
+	// map header, size 5
+	// write "UID"
+	err = en.Append(0x85, 0xa3, 0x55, 0x49, 0x44)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.UID)
+	if err != nil {
+		err = msgp.WrapError(err, "UID")
+		return
+	}
+	// write "Resources"
+	err = en.Append(0xa9, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73)
+	if err != nil {
+		return
+	}
+	err = en.WriteArrayHeader(uint32(len(z.Resources)))
+	if err != nil {
+		err = msgp.WrapError(err, "Resources")
+		return
+	}
+	for za0001 := range z.Resources {
+		err = en.WriteString(z.Resources[za0001])
+		if err != nil {
+			err = msgp.WrapError(err, "Resources", za0001)
+			return
+		}
+	}
+	// write "Owner"
+	err = en.Append(0xa5, 0x4f, 0x77, 0x6e, 0x65, 0x72)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.Owner)
+	if err != nil {
+		err = msgp.WrapError(err, "Owner")
+		return
+	}
+	// write "Source"
+	err = en.Append(0xa6, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.Source)
+	if err != nil {
+		err = msgp.WrapError(err, "Source")
+		return
+	}
+	// write "Quorum"
+	err = en.Append(0xa6, 0x51, 0x75, 0x6f, 0x72, 0x75, 0x6d)
+	if err != nil {
+		return
+	}
+	if z.Quorum == nil {
+		err = en.WriteNil()
+		if err != nil {
+			return
+		}
+	} else {
+		err = en.WriteInt(*z.Quorum)
+		if err != nil {
+			err = msgp.WrapError(err, "Quorum")
+			return
+		}
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z *LockArgs) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// map header, size 5
+	// string "UID"
+	o = append(o, 0x85, 0xa3, 0x55, 0x49, 0x44)
+	o = msgp.AppendString(o, z.UID)
+	// string "Resources"
+	o = append(o, 0xa9, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73)
+	o = msgp.AppendArrayHeader(o, uint32(len(z.Resources)))
+	for za0001 := range z.Resources {
+		o = msgp.AppendString(o, z.Resources[za0001])
+	}
+	// string "Owner"
+	o = append(o, 0xa5, 0x4f, 0x77, 0x6e, 0x65, 0x72)
+	o = msgp.AppendString(o, z.Owner)
+	// string "Source"
+	o = append(o, 0xa6, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65)
+	o = msgp.AppendString(o, z.Source)
+	// string "Quorum"
+	o = append(o, 0xa6, 0x51, 0x75, 0x6f, 0x72, 0x75, 0x6d)
+	if z.Quorum == nil {
+		o = msgp.AppendNil(o)
+	} else {
+		o = msgp.AppendInt(o, *z.Quorum)
+	}
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *LockArgs) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "UID":
+			z.UID, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "UID")
+				return
+			}
+		case "Resources":
+			var zb0002 uint32
+			zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Resources")
+				return
+			}
+			if cap(z.Resources) >= int(zb0002) {
+				z.Resources = (z.Resources)[:zb0002]
+			} else {
+				z.Resources = make([]string, zb0002)
+			}
+			for za0001 := range z.Resources {
+				z.Resources[za0001], bts, err = msgp.ReadStringBytes(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "Resources", za0001)
+					return
+				}
+			}
+		case "Owner":
+			z.Owner, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Owner")
+				return
+			}
+		case "Source":
+			z.Source, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Source")
+				return
+			}
+		case "Quorum":
+			if msgp.IsNil(bts) {
+				bts, err = msgp.ReadNilBytes(bts)
+				if err != nil {
+					return
+				}
+				z.Quorum = nil
+			} else {
+				if z.Quorum == nil {
+					z.Quorum = new(int)
+				}
+				*z.Quorum, bts, err = msgp.ReadIntBytes(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "Quorum")
+					return
+				}
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *LockArgs) Msgsize() (s int) {
+	s = 1 + 4 + msgp.StringPrefixSize + len(z.UID) + 10 + msgp.ArrayHeaderSize
+	for za0001 := range z.Resources {
+		s += msgp.StringPrefixSize + len(z.Resources[za0001])
+	}
+	s += 6 + msgp.StringPrefixSize + len(z.Owner) + 7 + msgp.StringPrefixSize + len(z.Source) + 7
+	if z.Quorum == nil {
+		s += msgp.NilSize
+	} else {
+		s += msgp.IntSize
+	}
+	return
+}
+
+// DecodeMsg implements msgp.Decodable
+func (z *LockResp) DecodeMsg(dc *msgp.Reader) (err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, err = dc.ReadMapHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, err = dc.ReadMapKeyPtr()
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "Code":
+			{
+				var zb0002 uint8
+				zb0002, err = dc.ReadUint8()
+				if err != nil {
+					err = msgp.WrapError(err, "Code")
+					return
+				}
+				z.Code = ResponseCode(zb0002)
+			}
+		case "Err":
+			z.Err, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "Err")
+				return
+			}
+		default:
+			err = dc.Skip()
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z LockResp) EncodeMsg(en *msgp.Writer) (err error) {
+	// map header, size 2
+	// write "Code"
+	err = en.Append(0x82, 0xa4, 0x43, 0x6f, 0x64, 0x65)
+	if err != nil {
+		return
+	}
+	err = en.WriteUint8(uint8(z.Code))
+	if err != nil {
+		err = msgp.WrapError(err, "Code")
+		return
+	}
+	// write "Err"
+	err = en.Append(0xa3, 0x45, 0x72, 0x72)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.Err)
+	if err != nil {
+		err = msgp.WrapError(err, "Err")
+		return
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z LockResp) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// map header, size 2
+	// string "Code"
+	o = append(o, 0x82, 0xa4, 0x43, 0x6f, 0x64, 0x65)
+	o = msgp.AppendUint8(o, uint8(z.Code))
+	// string "Err"
+	o = append(o, 0xa3, 0x45, 0x72, 0x72)
+	o = msgp.AppendString(o, z.Err)
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *LockResp) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "Code":
+			{
+				var zb0002 uint8
+				zb0002, bts, err = msgp.ReadUint8Bytes(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "Code")
+					return
+				}
+				z.Code = ResponseCode(zb0002)
+			}
+		case "Err":
+			z.Err, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Err")
+				return
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z LockResp) Msgsize() (s int) {
+	s = 1 + 5 + msgp.Uint8Size + 4 + msgp.StringPrefixSize + len(z.Err)
+	return
+}
+
+// DecodeMsg implements msgp.Decodable
+func (z *ResponseCode) DecodeMsg(dc *msgp.Reader) (err error) {
+	{
+		var zb0001 uint8
+		zb0001, err = dc.ReadUint8()
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		(*z) = ResponseCode(zb0001)
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z ResponseCode) EncodeMsg(en *msgp.Writer) (err error) {
+	err = en.WriteUint8(uint8(z))
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z ResponseCode) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	o = msgp.AppendUint8(o, uint8(z))
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *ResponseCode) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	{
+		var zb0001 uint8
+		zb0001, bts, err = msgp.ReadUint8Bytes(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		(*z) = ResponseCode(zb0001)
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z ResponseCode) Msgsize() (s int) {
+	s = msgp.Uint8Size
+	return
+}

internal/dsync/lock-args_gen_test.go

@@ -0,0 +1,236 @@
+// Code generated by github.com/tinylib/msgp DO NOT EDIT.
+
+package dsync
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/tinylib/msgp/msgp"
+)
+
+func TestMarshalUnmarshalLockArgs(t *testing.T) {
+	v := LockArgs{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgLockArgs(b *testing.B) {
+	v := LockArgs{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgLockArgs(b *testing.B) {
+	v := LockArgs{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalLockArgs(b *testing.B) {
+	v := LockArgs{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeLockArgs(t *testing.T) {
+	v := LockArgs{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeLockArgs Msgsize() is inaccurate")
+	}
+
+	vn := LockArgs{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeLockArgs(b *testing.B) {
+	v := LockArgs{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeLockArgs(b *testing.B) {
+	v := LockArgs{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalLockResp(t *testing.T) {
+	v := LockResp{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgLockResp(b *testing.B) {
+	v := LockResp{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgLockResp(b *testing.B) {
+	v := LockResp{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalLockResp(b *testing.B) {
+	v := LockResp{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeLockResp(t *testing.T) {
+	v := LockResp{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeLockResp Msgsize() is inaccurate")
+	}
+
+	vn := LockResp{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeLockResp(b *testing.B) {
+	v := LockResp{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeLockResp(b *testing.B) {
+	v := LockResp{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}

internal/dsync/locked_rand.go

@@ -0,0 +1,45 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package dsync
+
+import (
+	"math/rand"
+	"sync"
+)
+
+// lockedRandSource provides protected rand source, implements rand.Source interface.
+type lockedRandSource struct {
+	lk  sync.Mutex
+	src rand.Source
+}
+
+// Int63 returns a non-negative pseudo-random 63-bit integer as an int64.
+func (r *lockedRandSource) Int63() (n int64) {
+	r.lk.Lock()
+	n = r.src.Int63()
+	r.lk.Unlock()
+	return n
+}
+
+// Seed uses the provided seed value to initialize the generator to a
+// deterministic state.
+func (r *lockedRandSource) Seed(seed int64) {
+	r.lk.Lock()
+	r.src.Seed(seed)
+	r.lk.Unlock()
+}

internal/dsync/locker.go

@@ -0,0 +1,67 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package dsync
+
+import "context"
+
+// NetLocker is dsync compatible locker interface.
+type NetLocker interface {
+	// Do read lock for given LockArgs.  It should return
+	// * a boolean to indicate success/failure of the operation
+	// * an error on failure of lock request operation.
+	RLock(ctx context.Context, args LockArgs) (bool, error)
+
+	// Do write lock for given LockArgs. It should return
+	// * a boolean to indicate success/failure of the operation
+	// * an error on failure of lock request operation.
+	Lock(ctx context.Context, args LockArgs) (bool, error)
+
+	// Do read unlock for given LockArgs. It should return
+	// * a boolean to indicate success/failure of the operation
+	// * an error on failure of unlock request operation.
+	// Canceling the context will abort the remote call.
+	// In that case, the resource may or may not be unlocked.
+	RUnlock(ctx context.Context, args LockArgs) (bool, error)
+
+	// Do write unlock for given LockArgs. It should return
+	// * a boolean to indicate success/failure of the operation
+	// * an error on failure of unlock request operation.
+	// Canceling the context will abort the remote call.
+	// In that case, the resource may or may not be unlocked.
+	Unlock(ctx context.Context, args LockArgs) (bool, error)
+
+	// Refresh the given lock to prevent it from becoming stale
+	Refresh(ctx context.Context, args LockArgs) (bool, error)
+
+	// Unlock (read/write) forcefully for given LockArgs. It should return
+	// * a boolean to indicate success/failure of the operation
+	// * an error on failure of unlock request operation.
+	ForceUnlock(ctx context.Context, args LockArgs) (bool, error)
+
+	// Returns underlying endpoint of this lock client instance.
+	String() string
+
+	// Close closes any underlying connection to the service endpoint
+	Close() error
+
+	// Is the underlying connection online? (is always true for any local lockers)
+	IsOnline() bool
+
+	// Is the underlying locker local to this server?
+	IsLocal() bool
+}

internal/dsync/utils.go

@@ -0,0 +1,39 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package dsync
+
+import (
+	"math/rand"
+	"time"
+)
+
+func backoffWait(minSleep, unit, maxSleep time.Duration) func(*rand.Rand, uint) time.Duration {
+	if unit > time.Hour {
+		// Protect against integer overflow
+		panic("unit cannot exceed one hour")
+	}
+	return func(r *rand.Rand, attempt uint) time.Duration {
+		sleep := minSleep
+		sleep += unit * time.Duration(attempt)
+		if sleep > maxSleep {
+			sleep = maxSleep
+		}
+		sleep -= time.Duration(r.Float64() * float64(sleep))
+		return sleep
+	}
+}