OP#792 initial commit after fork

internal/store/batch.go

@@ -0,0 +1,140 @@
+// Copyright (c) 2023 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package store
+
+import (
+	"context"
+	"errors"
+	"sync"
+	"time"
+)
+
+// ErrBatchFull indicates that the batch is full
+var ErrBatchFull = errors.New("batch is full")
+
+const defaultCommitTimeout = 30 * time.Second
+
+// Batch represents an ordered batch
+type Batch[I any] struct {
+	items  []I
+	limit  uint32
+	store  Store[I]
+	quitCh chan struct{}
+
+	sync.Mutex
+}
+
+// BatchConfig represents the batch config
+type BatchConfig[I any] struct {
+	Limit         uint32
+	Store         Store[I]
+	CommitTimeout time.Duration
+	Log           logger
+}
+
+// Add adds the item to the batch
+func (b *Batch[I]) Add(item I) error {
+	b.Lock()
+	defer b.Unlock()
+
+	if b.isFull() {
+		if b.store == nil {
+			return ErrBatchFull
+		}
+		// commit batch to store
+		if err := b.commit(); err != nil {
+			return err
+		}
+	}
+
+	b.items = append(b.items, item)
+	return nil
+}
+
+// Len returns the no of items in the batch
+func (b *Batch[_]) Len() int {
+	b.Lock()
+	defer b.Unlock()
+
+	return len(b.items)
+}
+
+func (b *Batch[_]) isFull() bool {
+	return len(b.items) >= int(b.limit)
+}
+
+func (b *Batch[I]) commit() error {
+	switch len(b.items) {
+	case 0:
+		return nil
+	case 1:
+		_, err := b.store.Put(b.items[0])
+		return err
+	default:
+	}
+	if _, err := b.store.PutMultiple(b.items); err != nil {
+		return err
+	}
+	b.items = make([]I, 0, b.limit)
+	return nil
+}
+
+// Close commits the pending items and quits the goroutines
+func (b *Batch[I]) Close() error {
+	defer func() {
+		close(b.quitCh)
+	}()
+
+	b.Lock()
+	defer b.Unlock()
+	return b.commit()
+}
+
+// NewBatch creates a new batch
+func NewBatch[I any](config BatchConfig[I]) *Batch[I] {
+	if config.CommitTimeout == 0 {
+		config.CommitTimeout = defaultCommitTimeout
+	}
+	quitCh := make(chan struct{})
+	batch := &Batch[I]{
+		items:  make([]I, 0, config.Limit),
+		limit:  config.Limit,
+		store:  config.Store,
+		quitCh: quitCh,
+	}
+	if batch.store != nil {
+		go func() {
+			commitTicker := time.NewTicker(config.CommitTimeout)
+			defer commitTicker.Stop()
+			for {
+				select {
+				case <-commitTicker.C:
+				case <-batch.quitCh:
+					return
+				}
+				batch.Lock()
+				err := batch.commit()
+				batch.Unlock()
+				if err != nil {
+					config.Log(context.Background(), err, "")
+				}
+			}
+		}()
+	}
+	return batch
+}

internal/store/batch_test.go

@@ -0,0 +1,219 @@
+// Copyright (c) 2023 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package store
+
+import (
+	"context"
+	"sync"
+	"testing"
+	"time"
+)
+
+func TestBatchCommit(t *testing.T) {
+	defer func() {
+		if err := tearDownQueueStore(); err != nil {
+			t.Fatalf("Failed to tear down store; %v", err)
+		}
+	}()
+	store, err := setUpQueueStore(queueDir, 100)
+	if err != nil {
+		t.Fatalf("Failed to create a queue store; %v", err)
+	}
+
+	var limit uint32 = 100
+
+	batch := NewBatch[TestItem](BatchConfig[TestItem]{
+		Limit:         limit,
+		Store:         store,
+		CommitTimeout: 5 * time.Minute,
+		Log: func(ctx context.Context, err error, id string, errKind ...any) {
+			t.Log(err)
+		},
+	})
+	defer batch.Close()
+
+	for i := 0; i < int(limit); i++ {
+		if err := batch.Add(testItem); err != nil {
+			t.Fatalf("failed to add %v; %v", i, err)
+		}
+	}
+
+	batchLen := batch.Len()
+	if batchLen != int(limit) {
+		t.Fatalf("Expected batch.Len() %v; but got %v", limit, batchLen)
+	}
+
+	keys := store.List()
+	if len(keys) > 0 {
+		t.Fatalf("Expected empty store list but got len(list) %v", len(keys))
+	}
+	if err := batch.Add(testItem); err != nil {
+		t.Fatalf("unable to add to the batch; %v", err)
+	}
+	batchLen = batch.Len()
+	if batchLen != 1 {
+		t.Fatalf("expected batch length to be 1 but got %v", batchLen)
+	}
+	keys = store.List()
+	if len(keys) != 1 {
+		t.Fatalf("expected len(store.List())=1; but got %v", len(keys))
+	}
+	key := keys[0]
+	if !key.Compress {
+		t.Fatal("expected key.Compress=true; but got false")
+	}
+	if key.ItemCount != int(limit) {
+		t.Fatalf("expected key.ItemCount=%d; but got %v", limit, key.ItemCount)
+	}
+	items, err := store.GetMultiple(key)
+	if err != nil {
+		t.Fatalf("unable to read key %v; %v", key.String(), err)
+	}
+	if len(items) != int(limit) {
+		t.Fatalf("expected len(items)=%d; but got %v", limit, len(items))
+	}
+}
+
+func TestBatchCommitOnExit(t *testing.T) {
+	defer func() {
+		if err := tearDownQueueStore(); err != nil {
+			t.Fatalf("Failed to tear down store; %v", err)
+		}
+	}()
+	store, err := setUpQueueStore(queueDir, 100)
+	if err != nil {
+		t.Fatalf("Failed to create a queue store; %v", err)
+	}
+
+	var limit uint32 = 100
+
+	batch := NewBatch[TestItem](BatchConfig[TestItem]{
+		Limit:         limit,
+		Store:         store,
+		CommitTimeout: 5 * time.Minute,
+		Log: func(ctx context.Context, err error, id string, errKind ...any) {
+			t.Log([]any{err, id, errKind}...)
+		},
+	})
+
+	for i := 0; i < int(limit); i++ {
+		if err := batch.Add(testItem); err != nil {
+			t.Fatalf("failed to add %v; %v", i, err)
+		}
+	}
+
+	batch.Close()
+	time.Sleep(1 * time.Second)
+
+	batchLen := batch.Len()
+	if batchLen != 0 {
+		t.Fatalf("Expected batch.Len()=0; but got %v", batchLen)
+	}
+
+	keys := store.List()
+	if len(keys) != 1 {
+		t.Fatalf("expected len(store.List())=1; but got %v", len(keys))
+	}
+
+	key := keys[0]
+	if !key.Compress {
+		t.Fatal("expected key.Compress=true; but got false")
+	}
+	if key.ItemCount != int(limit) {
+		t.Fatalf("expected key.ItemCount=%d; but got %v", limit, key.ItemCount)
+	}
+	items, err := store.GetMultiple(key)
+	if err != nil {
+		t.Fatalf("unable to read key %v; %v", key.String(), err)
+	}
+	if len(items) != int(limit) {
+		t.Fatalf("expected len(items)=%d; but got %v", limit, len(items))
+	}
+}
+
+func TestBatchWithConcurrency(t *testing.T) {
+	defer func() {
+		if err := tearDownQueueStore(); err != nil {
+			t.Fatalf("Failed to tear down store; %v", err)
+		}
+	}()
+	store, err := setUpQueueStore(queueDir, 100)
+	if err != nil {
+		t.Fatalf("Failed to create a queue store; %v", err)
+	}
+
+	var limit uint32 = 100
+
+	batch := NewBatch[TestItem](BatchConfig[TestItem]{
+		Limit:         limit,
+		Store:         store,
+		CommitTimeout: 5 * time.Minute,
+		Log: func(ctx context.Context, err error, id string, errKind ...any) {
+			t.Log(err)
+		},
+	})
+	defer batch.Close()
+
+	var wg sync.WaitGroup
+	for i := 0; i < int(limit); i++ {
+		wg.Add(1)
+		go func(key int) {
+			defer wg.Done()
+			if err := batch.Add(testItem); err != nil {
+				t.Errorf("failed to add item %v; %v", key, err)
+				return
+			}
+		}(i)
+	}
+	wg.Wait()
+
+	batchLen := batch.Len()
+	if batchLen != int(limit) {
+		t.Fatalf("Expected batch.Len() %v; but got %v", limit, batchLen)
+	}
+
+	keys := store.List()
+	if len(keys) > 0 {
+		t.Fatalf("Expected empty store list but got len(list) %v", len(keys))
+	}
+	if err := batch.Add(testItem); err != nil {
+		t.Fatalf("unable to add to the batch; %v", err)
+	}
+	batchLen = batch.Len()
+	if batchLen != 1 {
+		t.Fatalf("expected batch length to be 1 but got %v", batchLen)
+	}
+	keys = store.List()
+	if len(keys) != 1 {
+		t.Fatalf("expected len(store.List())=1; but got %v", len(keys))
+	}
+	key := keys[0]
+	if !key.Compress {
+		t.Fatal("expected key.Compress=true; but got false")
+	}
+	if key.ItemCount != int(limit) {
+		t.Fatalf("expected key.ItemCount=%d; but got %v", limit, key.ItemCount)
+	}
+	items, err := store.GetMultiple(key)
+	if err != nil {
+		t.Fatalf("unable to read key %v; %v", key.String(), err)
+	}
+	if len(items) != int(limit) {
+		t.Fatalf("expected len(items)=%d; but got %v", limit, len(items))
+	}
+}

internal/store/queuestore.go

@@ -0,0 +1,350 @@
+// Copyright (c) 2015-2023 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package store
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"os"
+	"path/filepath"
+	"sort"
+	"sync"
+	"time"
+
+	"github.com/google/uuid"
+	jsoniter "github.com/json-iterator/go"
+	"github.com/klauspost/compress/s2"
+	"github.com/valyala/bytebufferpool"
+)
+
+const (
+	defaultLimit = 100000 // Default store limit.
+	defaultExt   = ".unknown"
+	compressExt  = ".snappy"
+)
+
+// errLimitExceeded error is sent when the maximum limit is reached.
+var errLimitExceeded = errors.New("the maximum store limit reached")
+
+// QueueStore - Filestore for persisting items.
+type QueueStore[_ any] struct {
+	sync.RWMutex
+	entryLimit uint64
+	directory  string
+	fileExt    string
+
+	entries map[string]int64 // key -> modtime as unix nano
+}
+
+// NewQueueStore - Creates an instance for QueueStore.
+func NewQueueStore[I any](directory string, limit uint64, ext string) *QueueStore[I] {
+	if limit == 0 {
+		limit = defaultLimit
+	}
+
+	if ext == "" {
+		ext = defaultExt
+	}
+
+	return &QueueStore[I]{
+		directory:  directory,
+		entryLimit: limit,
+		fileExt:    ext,
+		entries:    make(map[string]int64, limit),
+	}
+}
+
+// Open - Creates the directory if not present.
+func (store *QueueStore[_]) Open() error {
+	store.Lock()
+	defer store.Unlock()
+
+	if err := os.MkdirAll(store.directory, os.FileMode(0o770)); err != nil {
+		return err
+	}
+
+	files, err := store.list()
+	if err != nil {
+		return err
+	}
+
+	for _, file := range files {
+		if file.IsDir() {
+			continue
+		}
+		if fi, err := file.Info(); err == nil {
+			store.entries[file.Name()] = fi.ModTime().UnixNano()
+		}
+	}
+
+	return nil
+}
+
+// Delete - Remove the store directory from disk
+func (store *QueueStore[_]) Delete() error {
+	return os.Remove(store.directory)
+}
+
+// PutMultiple - puts an item to the store.
+func (store *QueueStore[I]) PutMultiple(items []I) (Key, error) {
+	// Generate a new UUID for the key.
+	uid, err := uuid.NewRandom()
+	if err != nil {
+		return Key{}, err
+	}
+
+	store.Lock()
+	defer store.Unlock()
+	if uint64(len(store.entries)) >= store.entryLimit {
+		return Key{}, errLimitExceeded
+	}
+	key := Key{
+		Name:      uid.String(),
+		ItemCount: len(items),
+		Compress:  true,
+		Extension: store.fileExt,
+	}
+	return key, store.multiWrite(key, items)
+}
+
+// multiWrite - writes an item to the directory.
+func (store *QueueStore[I]) multiWrite(key Key, items []I) (err error) {
+	buf := bytebufferpool.Get()
+	defer bytebufferpool.Put(buf)
+
+	enc := jsoniter.ConfigCompatibleWithStandardLibrary.NewEncoder(buf)
+
+	for i := range items {
+		if err = enc.Encode(items[i]); err != nil {
+			return err
+		}
+	}
+
+	path := filepath.Join(store.directory, key.String())
+	if key.Compress {
+		err = os.WriteFile(path, s2.Encode(nil, buf.Bytes()), os.FileMode(0o770))
+	} else {
+		err = os.WriteFile(path, buf.Bytes(), os.FileMode(0o770))
+	}
+
+	buf.Reset()
+	if err != nil {
+		return err
+	}
+
+	// Increment the item count.
+	store.entries[key.String()] = time.Now().UnixNano()
+
+	return err
+}
+
+// write - writes an item to the directory.
+func (store *QueueStore[I]) write(key Key, item I) error {
+	// Marshals the item.
+	eventData, err := json.Marshal(item)
+	if err != nil {
+		return err
+	}
+	return store.writeBytes(key, eventData)
+}
+
+// writeBytes - writes bytes to the directory.
+func (store *QueueStore[I]) writeBytes(key Key, b []byte) (err error) {
+	path := filepath.Join(store.directory, key.String())
+
+	if key.Compress {
+		err = os.WriteFile(path, s2.Encode(nil, b), os.FileMode(0o770))
+	} else {
+		err = os.WriteFile(path, b, os.FileMode(0o770))
+	}
+
+	if err != nil {
+		return err
+	}
+	// Increment the item count.
+	store.entries[key.String()] = time.Now().UnixNano()
+	return nil
+}
+
+// Put - puts an item to the store.
+func (store *QueueStore[I]) Put(item I) (Key, error) {
+	store.Lock()
+	defer store.Unlock()
+	if uint64(len(store.entries)) >= store.entryLimit {
+		return Key{}, errLimitExceeded
+	}
+	// Generate a new UUID for the key.
+	uid, err := uuid.NewRandom()
+	if err != nil {
+		return Key{}, err
+	}
+	key := Key{
+		Name:      uid.String(),
+		Extension: store.fileExt,
+		ItemCount: 1,
+	}
+	return key, store.write(key, item)
+}
+
+// PutRaw - puts the raw bytes to the store
+func (store *QueueStore[I]) PutRaw(b []byte) (Key, error) {
+	store.Lock()
+	defer store.Unlock()
+	if uint64(len(store.entries)) >= store.entryLimit {
+		return Key{}, errLimitExceeded
+	}
+	// Generate a new UUID for the key.
+	uid, err := uuid.NewRandom()
+	if err != nil {
+		return Key{}, err
+	}
+	key := Key{
+		Name:      uid.String(),
+		Extension: store.fileExt,
+	}
+	return key, store.writeBytes(key, b)
+}
+
+// GetRaw - gets an item from the store.
+func (store *QueueStore[I]) GetRaw(key Key) (raw []byte, err error) {
+	store.RLock()
+
+	defer func(store *QueueStore[I]) {
+		store.RUnlock()
+		if err != nil && !os.IsNotExist(err) {
+			// Upon error we remove the entry.
+			store.Del(key)
+		}
+	}(store)
+
+	raw, err = os.ReadFile(filepath.Join(store.directory, key.String()))
+	if err != nil {
+		return raw, err
+	}
+
+	if len(raw) == 0 {
+		return raw, os.ErrNotExist
+	}
+
+	if key.Compress {
+		raw, err = s2.Decode(nil, raw)
+	}
+
+	return raw, err
+}
+
+// Get - gets an item from the store.
+func (store *QueueStore[I]) Get(key Key) (item I, err error) {
+	items, err := store.GetMultiple(key)
+	if err != nil {
+		return item, err
+	}
+	return items[0], nil
+}
+
+// GetMultiple will read the multi payload file and fetch the items
+func (store *QueueStore[I]) GetMultiple(key Key) (items []I, err error) {
+	raw, err := store.GetRaw(key)
+	if err != nil {
+		return nil, err
+	}
+
+	decoder := jsoniter.ConfigCompatibleWithStandardLibrary.NewDecoder(bytes.NewReader(raw))
+	for decoder.More() {
+		var item I
+		if err := decoder.Decode(&item); err != nil {
+			return nil, err
+		}
+		items = append(items, item)
+	}
+
+	return items, err
+}
+
+// Del - Deletes an entry from the store.
+func (store *QueueStore[_]) Del(key Key) error {
+	store.Lock()
+	defer store.Unlock()
+	return store.del(key)
+}
+
+// Len returns the entry count.
+func (store *QueueStore[_]) Len() int {
+	store.RLock()
+	l := len(store.entries)
+	defer store.RUnlock()
+	return l
+}
+
+// lockless call
+func (store *QueueStore[_]) del(key Key) error {
+	err := os.Remove(filepath.Join(store.directory, key.String()))
+
+	// Delete as entry no matter the result
+	delete(store.entries, key.String())
+
+	return err
+}
+
+// List - lists all files registered in the store.
+func (store *QueueStore[_]) List() (keys []Key) {
+	store.RLock()
+	defer store.RUnlock()
+
+	entries := make([]string, 0, len(store.entries))
+	for entry := range store.entries {
+		entries = append(entries, entry)
+	}
+
+	// Sort entries...
+	sort.Slice(entries, func(i, j int) bool {
+		return store.entries[entries[i]] < store.entries[entries[j]]
+	})
+
+	for i := range entries {
+		keys = append(keys, parseKey(entries[i]))
+	}
+
+	return keys
+}
+
+// list will read all entries from disk.
+// Entries are returned sorted by modtime, oldest first.
+// Underlying entry list in store is *not* updated.
+func (store *QueueStore[_]) list() ([]os.DirEntry, error) {
+	files, err := os.ReadDir(store.directory)
+	if err != nil {
+		return nil, err
+	}
+
+	// Sort the entries.
+	sort.Slice(files, func(i, j int) bool {
+		ii, err := files[i].Info()
+		if err != nil {
+			return false
+		}
+		ji, err := files[j].Info()
+		if err != nil {
+			return true
+		}
+		return ii.ModTime().Before(ji.ModTime())
+	})
+
+	return files, nil
+}

internal/store/queuestore_test.go

@@ -0,0 +1,421 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package store
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+	"path/filepath"
+	"reflect"
+	"testing"
+
+	jsoniter "github.com/json-iterator/go"
+	"github.com/valyala/bytebufferpool"
+)
+
+type TestItem struct {
+	Name     string `json:"Name"`
+	Property string `json:"property"`
+}
+
+var (
+	// TestDir
+	queueDir = filepath.Join(os.TempDir(), "minio_test")
+	// Sample test item.
+	testItem = TestItem{Name: "test-item", Property: "property"}
+	// Ext for test item
+	testItemExt = ".test"
+)
+
+// Initialize the queue store.
+func setUpQueueStore(directory string, limit uint64) (Store[TestItem], error) {
+	queueStore := NewQueueStore[TestItem](queueDir, limit, testItemExt)
+	if oErr := queueStore.Open(); oErr != nil {
+		return nil, oErr
+	}
+	return queueStore, nil
+}
+
+// Tear down queue store.
+func tearDownQueueStore() error {
+	return os.RemoveAll(queueDir)
+}
+
+// TestQueueStorePut - tests for store.Put
+func TestQueueStorePut(t *testing.T) {
+	defer func() {
+		if err := tearDownQueueStore(); err != nil {
+			t.Fatal("Failed to tear down store ", err)
+		}
+	}()
+	store, err := setUpQueueStore(queueDir, 100)
+	if err != nil {
+		t.Fatal("Failed to create a queue store ", err)
+	}
+	// Put 100 items.
+	for range 100 {
+		if _, err := store.Put(testItem); err != nil {
+			t.Fatal("Failed to put to queue store ", err)
+		}
+	}
+	// Count the items.
+	keys := store.List()
+	if len(keys) != 100 {
+		t.Fatalf("List() Expected: 100, got %d", len(keys))
+	}
+}
+
+// TestQueueStoreGet - tests for store.Get
+func TestQueueStoreGet(t *testing.T) {
+	defer func() {
+		if err := tearDownQueueStore(); err != nil {
+			t.Fatal("Failed to tear down store ", err)
+		}
+	}()
+	store, err := setUpQueueStore(queueDir, 10)
+	if err != nil {
+		t.Fatal("Failed to create a queue store ", err)
+	}
+	// Put 10 items
+	for range 10 {
+		if _, err := store.Put(testItem); err != nil {
+			t.Fatal("Failed to put to queue store ", err)
+		}
+	}
+	itemKeys := store.List()
+	// Get 10 items.
+	if len(itemKeys) == 10 {
+		for _, key := range itemKeys {
+			item, eErr := store.Get(key)
+			if eErr != nil {
+				t.Fatal("Failed to Get the item from the queue store ", eErr)
+			}
+			if !reflect.DeepEqual(testItem, item) {
+				t.Fatalf("Failed to read the item: error: expected = %v, got = %v", testItem, item)
+			}
+		}
+	} else {
+		t.Fatalf("List() Expected: 10, got %d", len(itemKeys))
+	}
+}
+
+// TestQueueStoreDel - tests for store.Del
+func TestQueueStoreDel(t *testing.T) {
+	defer func() {
+		if err := tearDownQueueStore(); err != nil {
+			t.Fatal("Failed to tear down store ", err)
+		}
+	}()
+	store, err := setUpQueueStore(queueDir, 20)
+	if err != nil {
+		t.Fatal("Failed to create a queue store ", err)
+	}
+	// Put 20 items.
+	for range 20 {
+		if _, err := store.Put(testItem); err != nil {
+			t.Fatal("Failed to put to queue store ", err)
+		}
+	}
+	itemKeys := store.List()
+	// Remove all the items.
+	if len(itemKeys) == 20 {
+		for _, key := range itemKeys {
+			err := store.Del(key)
+			if err != nil {
+				t.Fatal("queue store Del failed with ", err)
+			}
+		}
+	} else {
+		t.Fatalf("List() Expected: 20, got %d", len(itemKeys))
+	}
+
+	keys := store.List()
+	if len(keys) != 0 {
+		t.Fatalf("List() Expected: 0, got %d", len(keys))
+	}
+}
+
+// TestQueueStoreLimit - tests the item limit for the store.
+func TestQueueStoreLimit(t *testing.T) {
+	defer func() {
+		if err := tearDownQueueStore(); err != nil {
+			t.Fatal("Failed to tear down store ", err)
+		}
+	}()
+	// The max limit is set to 5.
+	store, err := setUpQueueStore(queueDir, 5)
+	if err != nil {
+		t.Fatal("Failed to create a queue store ", err)
+	}
+	for range 5 {
+		if _, err := store.Put(testItem); err != nil {
+			t.Fatal("Failed to put to queue store ", err)
+		}
+	}
+	// Should not allow 6th Put.
+	if _, err := store.Put(testItem); err == nil {
+		t.Fatalf("Expected to fail with %s, but passes", errLimitExceeded)
+	}
+}
+
+// TestQueueStoreLimit - tests for store.LimitN.
+func TestQueueStoreListN(t *testing.T) {
+	defer func() {
+		if err := tearDownQueueStore(); err != nil {
+			t.Fatal("Failed to tear down store ", err)
+		}
+	}()
+	store, err := setUpQueueStore(queueDir, 10)
+	if err != nil {
+		t.Fatal("Failed to create a queue store ", err)
+	}
+	for range 10 {
+		if _, err := store.Put(testItem); err != nil {
+			t.Fatal("Failed to put to queue store ", err)
+		}
+	}
+	// Should return all the item keys in the store.
+	keys := store.List()
+
+	if len(keys) != 10 {
+		t.Fatalf("List() Expected: 10, got %d", len(keys))
+	}
+
+	// re-open
+	store, err = setUpQueueStore(queueDir, 10)
+	if err != nil {
+		t.Fatal("Failed to create a queue store ", err)
+	}
+	keys = store.List()
+
+	if len(keys) != 10 {
+		t.Fatalf("List() Expected: 10, got %d", len(keys))
+	}
+	if len(keys) != store.Len() {
+		t.Fatalf("List() Expected: 10, got %d", len(keys))
+	}
+
+	// Delete all
+	for _, key := range keys {
+		err := store.Del(key)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+	// Re-list
+	keys = store.List()
+	if len(keys) > 0 || err != nil {
+		t.Fatalf("Expected List() to return empty list and no error, got %v err: %v", keys, err)
+	}
+}
+
+func TestMultiplePutGetRaw(t *testing.T) {
+	defer func() {
+		if err := tearDownQueueStore(); err != nil {
+			t.Fatalf("Failed to tear down store; %v", err)
+		}
+	}()
+	store, err := setUpQueueStore(queueDir, 10)
+	if err != nil {
+		t.Fatalf("Failed to create a queue store; %v", err)
+	}
+	// TestItem{Name: "test-item", Property: "property"}
+	var items []TestItem
+	for i := range 10 {
+		items = append(items, TestItem{
+			Name:     fmt.Sprintf("test-item-%d", i),
+			Property: "property",
+		})
+	}
+
+	buf := bytebufferpool.Get()
+	defer bytebufferpool.Put(buf)
+
+	enc := jsoniter.ConfigCompatibleWithStandardLibrary.NewEncoder(buf)
+	for i := range items {
+		if err = enc.Encode(items[i]); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	if _, err := store.PutMultiple(items); err != nil {
+		t.Fatalf("failed to put multiple; %v", err)
+	}
+
+	keys := store.List()
+	if len(keys) != 1 {
+		t.Fatalf("expected len(keys)=1, but found %d", len(keys))
+	}
+
+	key := keys[0]
+	if !key.Compress {
+		t.Fatal("expected the item to be compressed")
+	}
+	if key.ItemCount != 10 {
+		t.Fatalf("expected itemcount=10 but found %v", key.ItemCount)
+	}
+
+	raw, err := store.GetRaw(key)
+	if err != nil {
+		t.Fatalf("unable to get multiple items; %v", err)
+	}
+
+	if !bytes.Equal(buf.Bytes(), raw) {
+		t.Fatalf("expected bytes: %d vs read bytes is wrong %d", len(buf.Bytes()), len(raw))
+	}
+
+	if err := store.Del(key); err != nil {
+		t.Fatalf("unable to Del; %v", err)
+	}
+
+	// Re-list
+	keys = store.List()
+	if len(keys) > 0 || err != nil {
+		t.Fatalf("Expected List() to return empty list and no error, got %v err: %v", keys, err)
+	}
+}
+
+func TestMultiplePutGets(t *testing.T) {
+	defer func() {
+		if err := tearDownQueueStore(); err != nil {
+			t.Fatalf("Failed to tear down store; %v", err)
+		}
+	}()
+	store, err := setUpQueueStore(queueDir, 10)
+	if err != nil {
+		t.Fatalf("Failed to create a queue store; %v", err)
+	}
+	// TestItem{Name: "test-item", Property: "property"}
+	var items []TestItem
+	for i := range 10 {
+		items = append(items, TestItem{
+			Name:     fmt.Sprintf("test-item-%d", i),
+			Property: "property",
+		})
+	}
+
+	if _, err := store.PutMultiple(items); err != nil {
+		t.Fatalf("failed to put multiple; %v", err)
+	}
+
+	keys := store.List()
+	if len(keys) != 1 {
+		t.Fatalf("expected len(keys)=1, but found %d", len(keys))
+	}
+
+	key := keys[0]
+	if !key.Compress {
+		t.Fatal("expected the item to be compressed")
+	}
+	if key.ItemCount != 10 {
+		t.Fatalf("expected itemcount=10 but found %v", key.ItemCount)
+	}
+
+	resultItems, err := store.GetMultiple(key)
+	if err != nil {
+		t.Fatalf("unable to get multiple items; %v", err)
+	}
+
+	if !reflect.DeepEqual(resultItems, items) {
+		t.Fatalf("expected item list: %v; but got %v", items, resultItems)
+	}
+
+	if err := store.Del(key); err != nil {
+		t.Fatalf("unable to Del; %v", err)
+	}
+
+	// Re-list
+	keys = store.List()
+	if len(keys) > 0 || err != nil {
+		t.Fatalf("Expected List() to return empty list and no error, got %v err: %v", keys, err)
+	}
+}
+
+func TestMixedPutGets(t *testing.T) {
+	defer func() {
+		if err := tearDownQueueStore(); err != nil {
+			t.Fatalf("Failed to tear down store; %v", err)
+		}
+	}()
+	store, err := setUpQueueStore(queueDir, 10)
+	if err != nil {
+		t.Fatalf("Failed to create a queue store; %v", err)
+	}
+	// TestItem{Name: "test-item", Property: "property"}
+	var items []TestItem
+	for i := range 5 {
+		items = append(items, TestItem{
+			Name:     fmt.Sprintf("test-item-%d", i),
+			Property: "property",
+		})
+	}
+	if _, err := store.PutMultiple(items); err != nil {
+		t.Fatalf("failed to put multiple; %v", err)
+	}
+
+	for i := 5; i < 10; i++ {
+		item := TestItem{
+			Name:     fmt.Sprintf("test-item-%d", i),
+			Property: "property",
+		}
+		if _, err := store.Put(item); err != nil {
+			t.Fatalf("unable to store.Put(); %v", err)
+		}
+		items = append(items, item)
+	}
+
+	keys := store.List()
+	if len(keys) != 6 {
+		// 1 multiple + 5 single PUTs
+		t.Fatalf("expected len(keys)=6, but found %d", len(keys))
+	}
+
+	var resultItems []TestItem
+	for _, key := range keys {
+		if key.ItemCount > 1 {
+			items, err := store.GetMultiple(key)
+			if err != nil {
+				t.Fatalf("unable to get multiple items; %v", err)
+			}
+			resultItems = append(resultItems, items...)
+			continue
+		}
+		item, err := store.Get(key)
+		if err != nil {
+			t.Fatalf("unable to get item; %v", err)
+		}
+		resultItems = append(resultItems, item)
+	}
+
+	if !reflect.DeepEqual(resultItems, items) {
+		t.Fatalf("expected item list: %v; but got %v", items, resultItems)
+	}
+
+	// Delete all
+	for _, key := range keys {
+		if err := store.Del(key); err != nil {
+			t.Fatalf("unable to Del; %v", err)
+		}
+	}
+	// Re-list
+	keys = store.List()
+	if len(keys) > 0 || err != nil {
+		t.Fatalf("Expected List() to return empty list and no error, got %v err: %v", keys, err)
+	}
+}

internal/store/store.go

@@ -0,0 +1,189 @@
+// Copyright (c) 2015-2023 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package store
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+
+	xioutil "github.com/minio/minio/internal/ioutil"
+)
+
+const (
+	retryInterval = 3 * time.Second
+)
+
+type logger = func(ctx context.Context, err error, id string, errKind ...any)
+
+// ErrNotConnected - indicates that the target connection is not active.
+var ErrNotConnected = errors.New("not connected to target server/service")
+
+// Target - store target interface
+type Target interface {
+	Name() string
+	SendFromStore(key Key) error
+}
+
+// Store - Used to persist items.
+type Store[I any] interface {
+	Put(item I) (Key, error)
+	PutMultiple(item []I) (Key, error)
+	Get(key Key) (I, error)
+	GetMultiple(key Key) ([]I, error)
+	GetRaw(key Key) ([]byte, error)
+	PutRaw(b []byte) (Key, error)
+	Len() int
+	List() []Key
+	Del(key Key) error
+	Open() error
+	Delete() error
+}
+
+// Key denotes the key present in the store.
+type Key struct {
+	Name      string
+	Compress  bool
+	Extension string
+	ItemCount int
+}
+
+// String returns the filepath name
+func (k Key) String() string {
+	keyStr := k.Name
+	if k.ItemCount > 1 {
+		keyStr = fmt.Sprintf("%d:%s", k.ItemCount, k.Name)
+	}
+	return keyStr + k.Extension + func() string {
+		if k.Compress {
+			return compressExt
+		}
+		return ""
+	}()
+}
+
+func getItemCount(k string) (count int, err error) {
+	count = 1
+	v := strings.Split(k, ":")
+	if len(v) == 2 {
+		return strconv.Atoi(v[0])
+	}
+	return count, err
+}
+
+func parseKey(k string) (key Key) {
+	key.Name = k
+	if strings.HasSuffix(k, compressExt) {
+		key.Compress = true
+		key.Name = strings.TrimSuffix(key.Name, compressExt)
+	}
+	if key.ItemCount, _ = getItemCount(k); key.ItemCount > 1 {
+		key.Name = strings.TrimPrefix(key.Name, fmt.Sprintf("%d:", key.ItemCount))
+	}
+	if vals := strings.Split(key.Name, "."); len(vals) == 2 {
+		key.Extension = "." + vals[1]
+		key.Name = strings.TrimSuffix(key.Name, key.Extension)
+	}
+	return key
+}
+
+// replayItems - Reads the items from the store and replays.
+func replayItems[I any](store Store[I], doneCh <-chan struct{}, log logger, id string) <-chan Key {
+	keyCh := make(chan Key)
+
+	go func() {
+		defer xioutil.SafeClose(keyCh)
+
+		retryTicker := time.NewTicker(retryInterval)
+		defer retryTicker.Stop()
+
+		for {
+			for _, key := range store.List() {
+				select {
+				case keyCh <- key:
+				// Get next key.
+				case <-doneCh:
+					return
+				}
+			}
+
+			select {
+			case <-retryTicker.C:
+			case <-doneCh:
+				return
+			}
+		}
+	}()
+
+	return keyCh
+}
+
+// sendItems - Reads items from the store and re-plays.
+func sendItems(target Target, keyCh <-chan Key, doneCh <-chan struct{}, logger logger) {
+	retryTicker := time.NewTicker(retryInterval)
+	defer retryTicker.Stop()
+
+	send := func(key Key) bool {
+		for {
+			err := target.SendFromStore(key)
+			if err == nil {
+				break
+			}
+
+			logger(
+				context.Background(),
+				fmt.Errorf("unable to send log entry to '%s' err '%w'", target.Name(), err),
+				target.Name(),
+			)
+
+			select {
+			// Retrying after 3secs back-off
+			case <-retryTicker.C:
+			case <-doneCh:
+				return false
+			}
+		}
+		return true
+	}
+
+	for {
+		select {
+		case key, ok := <-keyCh:
+			if !ok {
+				return
+			}
+
+			if !send(key) {
+				return
+			}
+		case <-doneCh:
+			return
+		}
+	}
+}
+
+// StreamItems reads the keys from the store and replays the corresponding item to the target.
+func StreamItems[I any](store Store[I], target Target, doneCh <-chan struct{}, logger logger) {
+	go func() {
+		keyCh := replayItems(store, doneCh, logger, target.Name())
+		sendItems(target, keyCh, doneCh, logger)
+	}()
+}

internal/store/store_test.go

@@ -0,0 +1,146 @@
+// Copyright (c) 2015-2024 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package store
+
+import (
+	"testing"
+)
+
+func TestKeyString(t *testing.T) {
+	testCases := []struct {
+		key            Key
+		expectedString string
+	}{
+		{
+			key: Key{
+				Name:      "01894394-d046-4783-ba0d-f1c6885790dc",
+				Extension: ".event",
+			},
+			expectedString: "01894394-d046-4783-ba0d-f1c6885790dc.event",
+		},
+		{
+			key: Key{
+				Name:      "01894394-d046-4783-ba0d-f1c6885790dc",
+				Compress:  true,
+				Extension: ".event",
+				ItemCount: 100,
+			},
+			expectedString: "100:01894394-d046-4783-ba0d-f1c6885790dc.event.snappy",
+		},
+		{
+			key: Key{
+				Name:      "01894394-d046-4783-ba0d-f1c6885790dc",
+				Extension: ".event",
+				ItemCount: 100,
+			},
+			expectedString: "100:01894394-d046-4783-ba0d-f1c6885790dc.event",
+		},
+		{
+			key: Key{
+				Name:      "01894394-d046-4783-ba0d-f1c6885790dc",
+				Compress:  true,
+				Extension: ".event",
+				ItemCount: 1,
+			},
+			expectedString: "01894394-d046-4783-ba0d-f1c6885790dc.event.snappy",
+		},
+		{
+			key: Key{
+				Name:      "01894394-d046-4783-ba0d-f1c6885790dc",
+				Extension: ".event",
+				ItemCount: 1,
+			},
+			expectedString: "01894394-d046-4783-ba0d-f1c6885790dc.event",
+		},
+	}
+
+	for i, testCase := range testCases {
+		if testCase.expectedString != testCase.key.String() {
+			t.Fatalf("case[%v]: key.String() Expected: %s, got %s", i, testCase.expectedString, testCase.key.String())
+		}
+	}
+}
+
+func TestParseKey(t *testing.T) {
+	testCases := []struct {
+		str         string
+		expectedKey Key
+	}{
+		{
+			str: "01894394-d046-4783-ba0d-f1c6885790dc.event",
+			expectedKey: Key{
+				Name:      "01894394-d046-4783-ba0d-f1c6885790dc",
+				Extension: ".event",
+				ItemCount: 1,
+			},
+		},
+		{
+			str: "100:01894394-d046-4783-ba0d-f1c6885790dc.event.snappy",
+			expectedKey: Key{
+				Name:      "01894394-d046-4783-ba0d-f1c6885790dc",
+				Compress:  true,
+				Extension: ".event",
+				ItemCount: 100,
+			},
+		},
+		{
+			str: "100:01894394-d046-4783-ba0d-f1c6885790dc.event",
+			expectedKey: Key{
+				Name:      "01894394-d046-4783-ba0d-f1c6885790dc",
+				Extension: ".event",
+				ItemCount: 100,
+			},
+		},
+		{
+			str: "01894394-d046-4783-ba0d-f1c6885790dc.event.snappy",
+			expectedKey: Key{
+				Name:      "01894394-d046-4783-ba0d-f1c6885790dc",
+				Compress:  true,
+				Extension: ".event",
+				ItemCount: 1,
+			},
+		},
+		{
+			str: "01894394-d046-4783-ba0d-f1c6885790dc.event",
+			expectedKey: Key{
+				Name:      "01894394-d046-4783-ba0d-f1c6885790dc",
+				Extension: ".event",
+				ItemCount: 1,
+			},
+		},
+	}
+
+	for i, testCase := range testCases {
+		key := parseKey(testCase.str)
+		if testCase.expectedKey.Name != key.Name {
+			t.Fatalf("case[%v]: Expected key.Name: %v, got %v", i, testCase.expectedKey.Name, key.Name)
+		}
+		if testCase.expectedKey.Compress != key.Compress {
+			t.Fatalf("case[%v]: Expected key.Compress: %v, got %v", i, testCase.expectedKey.Compress, key.Compress)
+		}
+		if testCase.expectedKey.Extension != key.Extension {
+			t.Fatalf("case[%v]: Expected key.Extension: %v, got %v", i, testCase.expectedKey.Extension, key.Extension)
+		}
+		if testCase.expectedKey.ItemCount != key.ItemCount {
+			t.Fatalf("case[%v]: Expected key.ItemCount: %v, got %v", i, testCase.expectedKey.ItemCount, key.ItemCount)
+		}
+		if testCase.expectedKey.String() != key.String() {
+			t.Fatalf("case[%v]: Expected key.String(): %v, got %v", i, testCase.expectedKey.String(), key.String())
+		}
+	}
+}