OP#792 initial commit after fork

This commit is contained in:
2026-09-27 11:03:48 +02:00
parent 1fc33f4f1f
commit 55b0890196
1338 changed files with 428609 additions and 57 deletions

View File

@@ -0,0 +1,32 @@
// Code generated by "stringer -type Action lifecycle.go"; DO NOT EDIT.
package lifecycle
import "strconv"
func _() {
// An "invalid array index" compiler error signifies that the constant values have changed.
// Re-run the stringer command to generate them again.
var x [1]struct{}
_ = x[NoneAction-0]
_ = x[DeleteAction-1]
_ = x[DeleteVersionAction-2]
_ = x[TransitionAction-3]
_ = x[TransitionVersionAction-4]
_ = x[DeleteRestoredAction-5]
_ = x[DeleteRestoredVersionAction-6]
_ = x[DeleteAllVersionsAction-7]
_ = x[DelMarkerDeleteAllVersionsAction-8]
_ = x[ActionCount-9]
}
const _Action_name = "NoneActionDeleteActionDeleteVersionActionTransitionActionTransitionVersionActionDeleteRestoredActionDeleteRestoredVersionActionDeleteAllVersionsActionDelMarkerDeleteAllVersionsActionActionCount"
var _Action_index = [...]uint8{0, 10, 22, 41, 57, 80, 100, 127, 150, 182, 193}
func (i Action) String() string {
if i < 0 || i >= Action(len(_Action_index)-1) {
return "Action(" + strconv.FormatInt(int64(i), 10) + ")"
}
return _Action_name[_Action_index[i]:_Action_index[i+1]]
}

View File

@@ -0,0 +1,105 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package lifecycle
import (
"encoding/xml"
)
var errDuplicateTagKey = Errorf("Duplicate Tag Keys are not allowed")
// And - a tag to combine a prefix and multiple tags for lifecycle configuration rule.
type And struct {
XMLName xml.Name `xml:"And"`
ObjectSizeGreaterThan int64 `xml:"ObjectSizeGreaterThan,omitempty"`
ObjectSizeLessThan int64 `xml:"ObjectSizeLessThan,omitempty"`
Prefix Prefix `xml:"Prefix,omitempty"`
Tags []Tag `xml:"Tag,omitempty"`
}
// isEmpty returns true if Tags field is null
func (a And) isEmpty() bool {
return len(a.Tags) == 0 && !a.Prefix.set &&
a.ObjectSizeGreaterThan == 0 && a.ObjectSizeLessThan == 0
}
// Validate - validates the And field
func (a And) Validate() error {
// > This is used in a Lifecycle Rule Filter to apply a logical AND to two or more predicates.
// ref: https://docs.aws.amazon.com/AmazonS3/latest/API/API_LifecycleRuleAndOperator.html
// i.e, predCount >= 2
var predCount int
if a.Prefix.set {
predCount++
}
predCount += len(a.Tags)
if a.ObjectSizeGreaterThan > 0 {
predCount++
}
if a.ObjectSizeLessThan > 0 {
predCount++
}
if predCount < 2 {
return errXMLNotWellFormed
}
if a.ContainsDuplicateTag() {
return errDuplicateTagKey
}
for _, t := range a.Tags {
if err := t.Validate(); err != nil {
return err
}
}
if a.ObjectSizeGreaterThan < 0 || a.ObjectSizeLessThan < 0 {
return errXMLNotWellFormed
}
return nil
}
// ContainsDuplicateTag - returns true if duplicate keys are present in And
func (a And) ContainsDuplicateTag() bool {
x := make(map[string]struct{}, len(a.Tags))
for _, t := range a.Tags {
if _, has := x[t.Key]; has {
return true
}
x[t.Key] = struct{}{}
}
return false
}
// BySize returns true when sz satisfies a
// ObjectSizeLessThan/ObjectSizeGreaterthan or a logical AND of these predicates
// Note: And combines size and other predicates like Tags, Prefix, etc. This
// method applies exclusively to size predicates only.
func (a And) BySize(sz int64) bool {
if a.ObjectSizeGreaterThan > 0 &&
sz <= a.ObjectSizeGreaterThan {
return false
}
if a.ObjectSizeLessThan > 0 &&
sz >= a.ObjectSizeLessThan {
return false
}
return true
}

View File

@@ -0,0 +1,74 @@
// Copyright (c) 2024 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package lifecycle
import (
"encoding/xml"
"time"
)
var errInvalidDaysDelMarkerExpiration = Errorf("Days must be a positive integer with DelMarkerExpiration")
// DelMarkerExpiration used to xml encode/decode ILM action by the same name
type DelMarkerExpiration struct {
XMLName xml.Name `xml:"DelMarkerExpiration"`
Days int `xml:"Days,omitempty"`
}
// Empty returns if a DelMarkerExpiration XML element is empty.
// Used to detect if lifecycle.Rule contained a DelMarkerExpiration element.
func (de DelMarkerExpiration) Empty() bool {
return de.Days == 0
}
// UnmarshalXML decodes a single XML element into a DelMarkerExpiration value
func (de *DelMarkerExpiration) UnmarshalXML(dec *xml.Decoder, start xml.StartElement) error {
type delMarkerExpiration DelMarkerExpiration
var dexp delMarkerExpiration
err := dec.DecodeElement(&dexp, &start)
if err != nil {
return err
}
if dexp.Days <= 0 {
return errInvalidDaysDelMarkerExpiration
}
*de = DelMarkerExpiration(dexp)
return nil
}
// MarshalXML encodes a DelMarkerExpiration value into an XML element
func (de DelMarkerExpiration) MarshalXML(enc *xml.Encoder, start xml.StartElement) error {
if de.Empty() {
return nil
}
type delMarkerExpiration DelMarkerExpiration
return enc.EncodeElement(delMarkerExpiration(de), start)
}
// NextDue returns upcoming DelMarkerExpiration date for obj if
// applicable, returns false otherwise.
func (de DelMarkerExpiration) NextDue(obj ObjectOpts) (time.Time, bool) {
if !obj.IsLatest || !obj.DeleteMarker {
return time.Time{}, false
}
return ExpectedExpiryTime(obj.ModTime, de.Days), true
}

View File

@@ -0,0 +1,63 @@
// Copyright (c) 2024 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package lifecycle
import (
"encoding/xml"
"fmt"
"testing"
)
func TestDelMarkerExpParseAndValidate(t *testing.T) {
tests := []struct {
xml string
err error
}{
{
xml: `<DelMarkerExpiration> <Days> 1 </Days> </DelMarkerExpiration>`,
err: nil,
},
{
xml: `<DelMarkerExpiration> <Days> -1 </Days> </DelMarkerExpiration>`,
err: errInvalidDaysDelMarkerExpiration,
},
}
for i, test := range tests {
t.Run(fmt.Sprintf("TestDelMarker-%d", i), func(t *testing.T) {
var dexp DelMarkerExpiration
var fail bool
err := xml.Unmarshal([]byte(test.xml), &dexp)
if test.err == nil {
if err != nil {
fail = true
}
} else {
if err == nil {
fail = true
}
if test.err.Error() != err.Error() {
fail = true
}
}
if fail {
t.Fatalf("Expected %v but got %v", test.err, err)
}
})
}
}

View File

@@ -0,0 +1,45 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package lifecycle
import (
"fmt"
)
// Error is the generic type for any error happening during tag
// parsing.
type Error struct {
err error
}
// Errorf - formats according to a format specifier and returns
// the string as a value that satisfies error of type tagging.Error
func Errorf(format string, a ...any) error {
return Error{err: fmt.Errorf(format, a...)}
}
// Unwrap the internal error.
func (e Error) Unwrap() error { return e.err }
// Error 'error' compatible method.
func (e Error) Error() string {
if e.err == nil {
return "lifecycle: cause <nil>"
}
return e.err.Error()
}

View File

@@ -0,0 +1,156 @@
// Copyright (c) 2015-2025 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package lifecycle
import (
"fmt"
"time"
objlock "github.com/minio/minio/internal/bucket/object/lock"
"github.com/minio/minio/internal/bucket/replication"
)
// Evaluator - evaluates lifecycle policy on objects for the given lifecycle
// configuration, lock retention configuration and replication configuration.
type Evaluator struct {
policy Lifecycle
lockRetention *objlock.Retention
replCfg *replication.Config
}
// NewEvaluator - creates a new evaluator with the given lifecycle
func NewEvaluator(policy Lifecycle) *Evaluator {
return &Evaluator{
policy: policy,
}
}
// WithLockRetention - sets the lock retention configuration for the evaluator
func (e *Evaluator) WithLockRetention(lr *objlock.Retention) *Evaluator {
e.lockRetention = lr
return e
}
// WithReplicationConfig - sets the replication configuration for the evaluator
func (e *Evaluator) WithReplicationConfig(rcfg *replication.Config) *Evaluator {
e.replCfg = rcfg
return e
}
// IsPendingReplication checks if the object is pending replication.
func (e *Evaluator) IsPendingReplication(obj ObjectOpts) bool {
if e.replCfg == nil {
return false
}
if e.replCfg.HasActiveRules(obj.Name, true) && !obj.VersionPurgeStatus.Empty() {
return true
}
return false
}
// IsObjectLocked checks if it is appropriate to remove an
// object according to locking configuration when this is lifecycle/ bucket quota asking.
// (copied over from enforceRetentionForDeletion)
func (e *Evaluator) IsObjectLocked(obj ObjectOpts) bool {
if e.lockRetention == nil || !e.lockRetention.LockEnabled {
return false
}
if obj.DeleteMarker {
return false
}
lhold := objlock.GetObjectLegalHoldMeta(obj.UserDefined)
if lhold.Status.Valid() && lhold.Status == objlock.LegalHoldOn {
return true
}
ret := objlock.GetObjectRetentionMeta(obj.UserDefined)
if ret.Mode.Valid() && (ret.Mode == objlock.RetCompliance || ret.Mode == objlock.RetGovernance) {
t, err := objlock.UTCNowNTP()
if err != nil {
// it is safe to assume that the object is locked when
// we can't get the current time
return true
}
if ret.RetainUntilDate.After(t) {
return true
}
}
return false
}
// eval will return a lifecycle event for each object in objs for a given time.
func (e *Evaluator) eval(objs []ObjectOpts, now time.Time) []Event {
events := make([]Event, len(objs))
var newerNoncurrentVersions int
loop:
for i, obj := range objs {
event := e.policy.eval(obj, now, newerNoncurrentVersions)
switch event.Action {
case DeleteAllVersionsAction, DelMarkerDeleteAllVersionsAction:
// Skip if bucket has object locking enabled; To prevent the
// possibility of violating an object retention on one of the
// noncurrent versions of this object.
if e.lockRetention != nil && e.lockRetention.LockEnabled {
event = Event{}
} else {
// No need to evaluate remaining versions' lifecycle
// events after DeleteAllVersionsAction*
events[i] = event
break loop
}
case DeleteVersionAction, DeleteRestoredVersionAction:
// Defensive code, should never happen
if obj.VersionID == "" {
event.Action = NoneAction
}
if e.IsObjectLocked(obj) {
event = Event{}
}
if e.IsPendingReplication(obj) {
event = Event{}
}
}
if !obj.IsLatest {
switch event.Action {
case DeleteVersionAction:
// this noncurrent version will be expired, nothing to add
default:
// this noncurrent version will be spared
newerNoncurrentVersions++
}
}
events[i] = event
}
return events
}
// Eval will return a lifecycle event for each object in objs
func (e *Evaluator) Eval(objs []ObjectOpts) ([]Event, error) {
if len(objs) == 0 {
return nil, nil
}
if len(objs) != objs[0].NumVersions {
return nil, fmt.Errorf("number of versions mismatch, expected %d, got %d", objs[0].NumVersions, len(objs))
}
return e.eval(objs, time.Now().UTC()), nil
}

View File

@@ -0,0 +1,183 @@
// Copyright (c) 2015-2025 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package lifecycle
import (
"fmt"
"testing"
"time"
"github.com/google/uuid"
)
func TestNewerNoncurrentVersions(t *testing.T) {
prepLifecycleCfg := func(tagKeys []string, retainVersions []int) Lifecycle {
var lc Lifecycle
for i := range retainVersions {
ruleID := fmt.Sprintf("rule-%d", i)
tag := Tag{
Key: tagKeys[i],
Value: "minio",
}
lc.Rules = append(lc.Rules, Rule{
ID: ruleID,
Status: "Enabled",
Filter: Filter{
Tag: tag,
set: true,
},
NoncurrentVersionExpiration: NoncurrentVersionExpiration{
NewerNoncurrentVersions: retainVersions[i],
set: true,
},
})
}
return lc
}
lc := prepLifecycleCfg([]string{"tag3", "tag4", "tag5"}, []int{3, 4, 5})
evaluator := NewEvaluator(lc)
tagKeys := []string{"tag3", "tag3", "tag3", "tag4", "tag4", "tag5", "tag5"}
verIDs := []string{
"0NdAikoUVNGEpCUuB9vl.XyoMftMXCSg", "19M6Z405yFZuYygnnU9jKzsOBamTZK_7", "0PmlJdFWi_9d6l_dAkWrrhP.bBgtFk6V", // spellchecker:disable-line
".MmRalFNNJyOLymgCtQ3.qsdoYpy8qkB", "Bjb4OlMW9Agx.Nrggh15iU6frGu2CLde", "ngBmUd_cVl6ckONI9XsKGpJjzimohrzZ", // spellchecker:disable-line
"T6m1heTHLUtnByW2IOWJ3zM4JP9xXt2O", // spellchecker:disable-line
}
wantEvents := []Event{
{Action: NoneAction},
{Action: NoneAction},
{Action: NoneAction},
{Action: NoneAction},
{Action: NoneAction},
{Action: NoneAction},
{Action: DeleteVersionAction},
}
var objs []ObjectOpts
curModTime := time.Date(2025, time.February, 10, 23, 0, 0, 0, time.UTC)
for i := range tagKeys {
obj := ObjectOpts{
Name: "obj",
VersionID: verIDs[i],
ModTime: curModTime.Add(time.Duration(-i) * time.Second),
UserTags: fmt.Sprintf("%s=minio", tagKeys[i]),
NumVersions: len(verIDs),
}
if i == 0 {
obj.IsLatest = true
} else {
obj.SuccessorModTime = curModTime.Add(time.Duration(-i+1) * time.Second)
}
objs = append(objs, obj)
}
now := time.Date(2025, time.February, 10, 23, 0, 0, 0, time.UTC)
gotEvents := evaluator.eval(objs, now)
for i := range wantEvents {
if gotEvents[i].Action != wantEvents[i].Action {
t.Fatalf("got %v, want %v", gotEvents[i], wantEvents[i])
}
}
lc = prepLifecycleCfg([]string{"tag3", "tag4", "tag5"}, []int{1, 2, 3})
objs = objs[:len(objs)-1]
wantEvents = []Event{
{Action: NoneAction},
{Action: NoneAction},
{Action: DeleteVersionAction},
{Action: NoneAction},
{Action: DeleteVersionAction},
{Action: NoneAction},
}
evaluator = NewEvaluator(lc)
gotEvents = evaluator.eval(objs, now)
for i := range wantEvents {
if gotEvents[i].Action != wantEvents[i].Action {
t.Fatalf("test-%d: got %v, want %v", i+1, gotEvents[i], wantEvents[i])
}
}
lc = Lifecycle{
Rules: []Rule{
{
ID: "AllVersionsExpiration",
Status: "Enabled",
Filter: Filter{},
Expiration: Expiration{
Days: 1,
DeleteAll: Boolean{
val: true,
set: true,
},
set: true,
},
},
},
}
now = time.Date(2025, time.February, 12, 23, 0, 0, 0, time.UTC)
evaluator = NewEvaluator(lc)
gotEvents = evaluator.eval(objs, now)
wantEvents = []Event{
{Action: DeleteAllVersionsAction},
{Action: NoneAction},
{Action: NoneAction},
{Action: NoneAction},
{Action: NoneAction},
{Action: NoneAction},
}
for i := range wantEvents {
if gotEvents[i].Action != wantEvents[i].Action {
t.Fatalf("test-%d: got %v, want %v", i+1, gotEvents[i], wantEvents[i])
}
}
// Test with zero versions
events, err := evaluator.Eval(nil)
if len(events) != 0 || err != nil {
t.Fatal("expected no events nor error")
}
}
func TestEmptyEvaluator(t *testing.T) {
var objs []ObjectOpts
curModTime := time.Date(2025, time.February, 10, 23, 0, 0, 0, time.UTC)
for i := range 5 {
obj := ObjectOpts{
Name: "obj",
VersionID: uuid.New().String(),
ModTime: curModTime.Add(time.Duration(-i) * time.Second),
NumVersions: 5,
}
if i == 0 {
obj.IsLatest = true
} else {
obj.SuccessorModTime = curModTime.Add(time.Duration(-i+1) * time.Second)
}
objs = append(objs, obj)
}
evaluator := NewEvaluator(Lifecycle{})
events, err := evaluator.Eval(objs)
if err != nil {
t.Fatal(err)
}
for _, event := range events {
if event.Action != NoneAction {
t.Fatalf("got %v, want %v", event.Action, NoneAction)
}
}
}

View File

@@ -0,0 +1,211 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package lifecycle
import (
"encoding/xml"
"time"
)
var (
errLifecycleInvalidDate = Errorf("Date must be provided in ISO 8601 format")
errLifecycleInvalidDays = Errorf("Days must be positive integer when used with Expiration")
errLifecycleInvalidExpiration = Errorf("Exactly one of Days (positive integer) or Date (positive ISO 8601 format) should be present inside Expiration.")
errLifecycleInvalidDeleteMarker = Errorf("Delete marker cannot be specified with Days or Date in a Lifecycle Expiration Policy")
errLifecycleDateNotMidnight = Errorf("'Date' must be at midnight GMT")
errLifecycleInvalidDeleteAll = Errorf("Days (positive integer) should be present inside Expiration with ExpiredObjectAllVersions.")
)
// ExpirationDays is a type alias to unmarshal Days in Expiration
type ExpirationDays int
// UnmarshalXML parses number of days from Expiration and validates if
// greater than zero
func (eDays *ExpirationDays) UnmarshalXML(d *xml.Decoder, startElement xml.StartElement) error {
var numDays int
err := d.DecodeElement(&numDays, &startElement)
if err != nil {
return err
}
if numDays <= 0 {
return errLifecycleInvalidDays
}
*eDays = ExpirationDays(numDays)
return nil
}
// MarshalXML encodes number of days to expire if it is non-zero and
// encodes empty string otherwise
func (eDays ExpirationDays) MarshalXML(e *xml.Encoder, startElement xml.StartElement) error {
if eDays == 0 {
return nil
}
return e.EncodeElement(int(eDays), startElement)
}
// ExpirationDate is a embedded type containing time.Time to unmarshal
// Date in Expiration
type ExpirationDate struct {
time.Time
}
// UnmarshalXML parses date from Expiration and validates date format
func (eDate *ExpirationDate) UnmarshalXML(d *xml.Decoder, startElement xml.StartElement) error {
var dateStr string
err := d.DecodeElement(&dateStr, &startElement)
if err != nil {
return err
}
// While AWS documentation mentions that the date specified
// must be present in ISO 8601 format, in reality they allow
// users to provide RFC 3339 compliant dates.
expDate, err := time.Parse(time.RFC3339, dateStr)
if err != nil {
return errLifecycleInvalidDate
}
// Allow only date timestamp specifying midnight GMT
hr, m, sec := expDate.Clock()
nsec := expDate.Nanosecond()
loc := expDate.Location()
if hr != 0 || m != 0 || sec != 0 || nsec != 0 || loc.String() != time.UTC.String() {
return errLifecycleDateNotMidnight
}
*eDate = ExpirationDate{expDate}
return nil
}
// MarshalXML encodes expiration date if it is non-zero and encodes
// empty string otherwise
func (eDate ExpirationDate) MarshalXML(e *xml.Encoder, startElement xml.StartElement) error {
if eDate.IsZero() {
return nil
}
return e.EncodeElement(eDate.Format(time.RFC3339), startElement)
}
// ExpireDeleteMarker represents value of ExpiredObjectDeleteMarker field in Expiration XML element.
type ExpireDeleteMarker struct {
Boolean
}
// Boolean signifies a boolean XML struct with custom marshaling
type Boolean struct {
val bool
set bool
Unused struct{} // Needed for GOB compatibility
}
// Expiration - expiration actions for a rule in lifecycle configuration.
type Expiration struct {
XMLName xml.Name `xml:"Expiration"`
Days ExpirationDays `xml:"Days,omitempty"`
Date ExpirationDate `xml:"Date,omitempty"`
DeleteMarker ExpireDeleteMarker `xml:"ExpiredObjectDeleteMarker"`
// Indicates whether MinIO will remove all versions. If set to true, all versions will be deleted;
// if set to false the policy takes no action. This action uses the Days/Date to expire objects.
// This check is verified for latest version of the object.
DeleteAll Boolean `xml:"ExpiredObjectAllVersions"`
set bool
}
// MarshalXML encodes delete marker boolean into an XML form.
func (b Boolean) MarshalXML(e *xml.Encoder, startElement xml.StartElement) error {
if !b.set {
return nil
}
return e.EncodeElement(b.val, startElement)
}
// UnmarshalXML decodes delete marker boolean from the XML form.
func (b *Boolean) UnmarshalXML(d *xml.Decoder, startElement xml.StartElement) error {
var exp bool
err := d.DecodeElement(&exp, &startElement)
if err != nil {
return err
}
b.val = exp
b.set = true
return nil
}
// MarshalXML encodes expiration field into an XML form.
func (e Expiration) MarshalXML(enc *xml.Encoder, startElement xml.StartElement) error {
if !e.set {
return nil
}
type expirationWrapper Expiration
return enc.EncodeElement(expirationWrapper(e), startElement)
}
// UnmarshalXML decodes expiration field from the XML form.
func (e *Expiration) UnmarshalXML(d *xml.Decoder, startElement xml.StartElement) error {
type expirationWrapper Expiration
var exp expirationWrapper
err := d.DecodeElement(&exp, &startElement)
if err != nil {
return err
}
*e = Expiration(exp)
e.set = true
return nil
}
// Validate - validates the "Expiration" element
func (e Expiration) Validate() error {
if !e.set {
return nil
}
// DeleteMarker cannot be specified if date or dates are specified.
if (!e.IsDaysNull() || !e.IsDateNull()) && e.DeleteMarker.set {
return errLifecycleInvalidDeleteMarker
}
if !e.DeleteMarker.set && !e.DeleteAll.set && e.IsDaysNull() && e.IsDateNull() {
return errXMLNotWellFormed
}
// Both expiration days and date are specified
if !e.IsDaysNull() && !e.IsDateNull() {
return errLifecycleInvalidExpiration
}
// DeleteAll set without expiration days
if e.DeleteAll.set && e.IsDaysNull() {
return errLifecycleInvalidDeleteAll
}
return nil
}
// IsDaysNull returns true if days field is null
func (e Expiration) IsDaysNull() bool {
return e.Days == ExpirationDays(0)
}
// IsDateNull returns true if date field is null
func (e Expiration) IsDateNull() bool {
return e.Date.IsZero()
}
// IsNull returns true if both date and days fields are null
func (e Expiration) IsNull() bool {
return e.IsDaysNull() && e.IsDateNull()
}

View File

@@ -0,0 +1,126 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package lifecycle
import (
"encoding/xml"
"fmt"
"testing"
)
// appropriate errors on validation
func TestInvalidExpiration(t *testing.T) {
testCases := []struct {
inputXML string
expectedErr error
}{
{ // Expiration with zero days
inputXML: ` <Expiration>
<Days>0</Days>
</Expiration>`,
expectedErr: errLifecycleInvalidDays,
},
{ // Expiration with invalid date
inputXML: ` <Expiration>
<Date>invalid date</Date>
</Expiration>`,
expectedErr: errLifecycleInvalidDate,
},
{ // Expiration with both number of days nor a date
inputXML: `<Expiration>
<Date>2019-04-20T00:01:00Z</Date>
</Expiration>`,
expectedErr: errLifecycleDateNotMidnight,
},
}
for i, tc := range testCases {
t.Run(fmt.Sprintf("Test %d", i+1), func(t *testing.T) {
var expiration Expiration
err := xml.Unmarshal([]byte(tc.inputXML), &expiration)
if err != tc.expectedErr {
t.Fatalf("%d: Expected %v but got %v", i+1, tc.expectedErr, err)
}
})
}
validationTestCases := []struct {
inputXML string
expectedErr error
}{
{ // Expiration with a valid ISO 8601 date
inputXML: `<Expiration>
<Date>2019-04-20T00:00:00Z</Date>
</Expiration>`,
expectedErr: nil,
},
{ // Expiration with a valid number of days
inputXML: `<Expiration>
<Days>3</Days>
</Expiration>`,
expectedErr: nil,
},
{ // Expiration with neither number of days nor a date
inputXML: `<Expiration>
</Expiration>`,
expectedErr: errXMLNotWellFormed,
},
{ // Expiration with both number of days and a date
inputXML: `<Expiration>
<Days>3</Days>
<Date>2019-04-20T00:00:00Z</Date>
</Expiration>`,
expectedErr: errLifecycleInvalidExpiration,
},
{ // Expiration with both ExpiredObjectDeleteMarker and days
inputXML: `<Expiration>
<Days>3</Days>
<ExpiredObjectDeleteMarker>false</ExpiredObjectDeleteMarker>
</Expiration>`,
expectedErr: errLifecycleInvalidDeleteMarker,
},
{ // Expiration with a valid number of days and ExpiredObjectAllVersions
inputXML: `<Expiration>
<Days>3</Days>
<ExpiredObjectAllVersions>true</ExpiredObjectAllVersions>
</Expiration>`,
expectedErr: nil,
},
{ // Expiration with a valid ISO 8601 date and ExpiredObjectAllVersions
inputXML: `<Expiration>
<Date>2019-04-20T00:00:00Z</Date>
<ExpiredObjectAllVersions>true</ExpiredObjectAllVersions>
</Expiration>`,
expectedErr: errLifecycleInvalidDeleteAll,
},
}
for i, tc := range validationTestCases {
t.Run(fmt.Sprintf("Test %d", i+1), func(t *testing.T) {
var expiration Expiration
err := xml.Unmarshal([]byte(tc.inputXML), &expiration)
if err != nil {
t.Fatalf("%d: %v", i+1, err)
}
err = expiration.Validate()
if err != tc.expectedErr {
t.Fatalf("%d: got: %v, expected: %v", i+1, err, tc.expectedErr)
}
})
}
}

View File

@@ -0,0 +1,270 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package lifecycle
import (
"encoding/xml"
"io"
"github.com/minio/minio-go/v7/pkg/tags"
)
var errInvalidFilter = Errorf("Filter must have exactly one of Prefix, Tag, or And specified")
// Filter - a filter for a lifecycle configuration Rule.
type Filter struct {
XMLName xml.Name `xml:"Filter"`
set bool
Prefix Prefix
ObjectSizeGreaterThan int64 `xml:"ObjectSizeGreaterThan,omitempty"`
ObjectSizeLessThan int64 `xml:"ObjectSizeLessThan,omitempty"`
And And
andSet bool
Tag Tag
tagSet bool
// Caching tags, only once
cachedTags map[string]string
}
// MarshalXML - produces the xml representation of the Filter struct
// only one of Prefix, And and Tag should be present in the output.
func (f Filter) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
if !f.set {
return nil
}
if err := e.EncodeToken(start); err != nil {
return err
}
switch {
case !f.And.isEmpty():
if err := e.EncodeElement(f.And, xml.StartElement{Name: xml.Name{Local: "And"}}); err != nil {
return err
}
case !f.Tag.IsEmpty():
if err := e.EncodeElement(f.Tag, xml.StartElement{Name: xml.Name{Local: "Tag"}}); err != nil {
return err
}
default:
// Always print Prefix field when both And & Tag are empty
if err := e.EncodeElement(f.Prefix, xml.StartElement{Name: xml.Name{Local: "Prefix"}}); err != nil {
return err
}
if f.ObjectSizeLessThan > 0 {
if err := e.EncodeElement(f.ObjectSizeLessThan, xml.StartElement{Name: xml.Name{Local: "ObjectSizeLessThan"}}); err != nil {
return err
}
}
if f.ObjectSizeGreaterThan > 0 {
if err := e.EncodeElement(f.ObjectSizeGreaterThan, xml.StartElement{Name: xml.Name{Local: "ObjectSizeGreaterThan"}}); err != nil {
return err
}
}
}
return e.EncodeToken(xml.EndElement{Name: start.Name})
}
// UnmarshalXML - decodes XML data.
func (f *Filter) UnmarshalXML(d *xml.Decoder, start xml.StartElement) (err error) {
f.set = true
for {
// Read tokens from the XML document in a stream.
t, err := d.Token()
if err != nil {
if err == io.EOF {
break
}
return err
}
if se, ok := t.(xml.StartElement); ok {
switch se.Name.Local {
case "Prefix":
var p Prefix
if err = d.DecodeElement(&p, &se); err != nil {
return err
}
f.Prefix = p
case "And":
var and And
if err = d.DecodeElement(&and, &se); err != nil {
return err
}
f.And = and
f.andSet = true
case "Tag":
var tag Tag
if err = d.DecodeElement(&tag, &se); err != nil {
return err
}
f.Tag = tag
f.tagSet = true
case "ObjectSizeLessThan":
var sz int64
if err = d.DecodeElement(&sz, &se); err != nil {
return err
}
f.ObjectSizeLessThan = sz
case "ObjectSizeGreaterThan":
var sz int64
if err = d.DecodeElement(&sz, &se); err != nil {
return err
}
f.ObjectSizeGreaterThan = sz
default:
return errUnknownXMLTag
}
}
}
return nil
}
// IsEmpty returns true if Filter is not specified in the XML
func (f Filter) IsEmpty() bool {
return !f.set
}
// Validate - validates the filter element
func (f Filter) Validate() error {
if f.IsEmpty() {
return errXMLNotWellFormed
}
// A Filter must have exactly one of Prefix, Tag,
// ObjectSize{LessThan,GreaterThan} or And specified.
type predType uint8
const (
nonePred predType = iota
prefixPred
andPred
tagPred
sizeLtPred
sizeGtPred
)
var predCount int
var pType predType
if !f.And.isEmpty() {
pType = andPred
predCount++
}
if f.Prefix.set {
pType = prefixPred
predCount++
}
if !f.Tag.IsEmpty() {
pType = tagPred
predCount++
}
if f.ObjectSizeGreaterThan != 0 {
pType = sizeGtPred
predCount++
}
if f.ObjectSizeLessThan != 0 {
pType = sizeLtPred
predCount++
}
// Note: S3 supports empty <Filter></Filter>, so predCount == 0 is
// valid.
if predCount > 1 {
return errInvalidFilter
}
var err error
switch pType {
case nonePred:
// S3 supports empty <Filter></Filter>
case prefixPred:
case andPred:
err = f.And.Validate()
case tagPred:
err = f.Tag.Validate()
case sizeLtPred:
if f.ObjectSizeLessThan < 0 {
err = errXMLNotWellFormed
}
case sizeGtPred:
if f.ObjectSizeGreaterThan < 0 {
err = errXMLNotWellFormed
}
}
return err
}
// TestTags tests if the object tags satisfy the Filter tags requirement,
// it returns true if there is no tags in the underlying Filter.
func (f Filter) TestTags(userTags string) bool {
if f.cachedTags == nil {
cache := make(map[string]string)
for _, t := range append(f.And.Tags, f.Tag) {
if !t.IsEmpty() {
cache[t.Key] = t.Value
}
}
f.cachedTags = cache
}
// This filter does not have any tags, always return true
if len(f.cachedTags) == 0 {
return true
}
parsedTags, err := tags.ParseObjectTags(userTags)
if err != nil {
return false
}
tagsMap := parsedTags.ToMap()
// Not enough tags on object to satisfy the rule filter's tags
if len(tagsMap) < len(f.cachedTags) {
return false
}
var mismatch bool
for k, cv := range f.cachedTags {
v, ok := tagsMap[k]
if !ok || v != cv {
mismatch = true
break
}
}
return !mismatch
}
// BySize returns true if sz satisfies one of ObjectSizeGreaterThan,
// ObjectSizeLessThan predicates or a combination of them via And.
func (f Filter) BySize(sz int64) bool {
if f.ObjectSizeGreaterThan > 0 &&
sz <= f.ObjectSizeGreaterThan {
return false
}
if f.ObjectSizeLessThan > 0 &&
sz >= f.ObjectSizeLessThan {
return false
}
if !f.And.isEmpty() {
return f.And.BySize(sz)
}
return true
}

View File

@@ -0,0 +1,343 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package lifecycle
import (
"encoding/xml"
"fmt"
"testing"
"github.com/dustin/go-humanize"
)
// TestUnsupportedFilters checks if parsing Filter xml with
// unsupported elements returns appropriate errors
func TestUnsupportedFilters(t *testing.T) {
testCases := []struct {
inputXML string
expectedErr error
}{
{ // Filter with And tags
inputXML: ` <Filter>
<And>
<Prefix>key-prefix</Prefix>
</And>
</Filter>`,
expectedErr: errXMLNotWellFormed,
},
{ // Filter with Tag tags
inputXML: ` <Filter>
<Tag>
<Key>key1</Key>
<Value>value1</Value>
</Tag>
</Filter>`,
expectedErr: nil,
},
{ // Filter with Prefix tag
inputXML: ` <Filter>
<Prefix>key-prefix</Prefix>
</Filter>`,
expectedErr: nil,
},
{ // Filter without And and multiple Tag tags
inputXML: ` <Filter>
<Prefix>key-prefix</Prefix>
<Tag>
<Key>key1</Key>
<Value>value1</Value>
</Tag>
<Tag>
<Key>key2</Key>
<Value>value2</Value>
</Tag>
</Filter>`,
expectedErr: errInvalidFilter,
},
{ // Filter with And, Prefix & multiple Tag tags
inputXML: ` <Filter>
<And>
<Prefix>key-prefix</Prefix>
<Tag>
<Key>key1</Key>
<Value>value1</Value>
</Tag>
<Tag>
<Key>key2</Key>
<Value>value2</Value>
</Tag>
</And>
</Filter>`,
expectedErr: nil,
},
{ // Filter with And and multiple Tag tags
inputXML: ` <Filter>
<And>
<Prefix></Prefix>
<Tag>
<Key>key1</Key>
<Value>value1</Value>
</Tag>
<Tag>
<Key>key2</Key>
<Value>value2</Value>
</Tag>
</And>
</Filter>`,
expectedErr: nil,
},
{ // Filter without And and single Tag tag
inputXML: ` <Filter>
<Prefix>key-prefix</Prefix>
<Tag>
<Key>key1</Key>
<Value>value1</Value>
</Tag>
</Filter>`,
expectedErr: errInvalidFilter,
},
}
for i, tc := range testCases {
t.Run(fmt.Sprintf("Test %d", i+1), func(t *testing.T) {
var filter Filter
err := xml.Unmarshal([]byte(tc.inputXML), &filter)
if err != nil {
t.Fatalf("%d: Expected no error but got %v", i+1, err)
}
err = filter.Validate()
if err != tc.expectedErr {
t.Fatalf("%d: Expected %v but got %v", i+1, tc.expectedErr, err)
}
})
}
}
func TestObjectSizeFilters(t *testing.T) {
f1 := Filter{
set: true,
Prefix: Prefix{
string: "doc/",
set: true,
Unused: struct{}{},
},
ObjectSizeGreaterThan: 100 * humanize.MiByte,
ObjectSizeLessThan: 100 * humanize.GiByte,
}
b, err := xml.Marshal(f1)
if err != nil {
t.Fatalf("Failed to marshal %v", f1)
}
var f2 Filter
err = xml.Unmarshal(b, &f2)
if err != nil {
t.Fatalf("Failed to unmarshal %s", string(b))
}
if f1.ObjectSizeLessThan != f2.ObjectSizeLessThan {
t.Fatalf("Expected %v but got %v", f1.ObjectSizeLessThan, f2.And.ObjectSizeLessThan)
}
if f1.ObjectSizeGreaterThan != f2.ObjectSizeGreaterThan {
t.Fatalf("Expected %v but got %v", f1.ObjectSizeGreaterThan, f2.And.ObjectSizeGreaterThan)
}
f1 = Filter{
set: true,
And: And{
ObjectSizeGreaterThan: 100 * humanize.MiByte,
ObjectSizeLessThan: 1 * humanize.GiByte,
Prefix: Prefix{},
},
andSet: true,
}
b, err = xml.Marshal(f1)
if err != nil {
t.Fatalf("Failed to marshal %v", f1)
}
f2 = Filter{}
err = xml.Unmarshal(b, &f2)
if err != nil {
t.Fatalf("Failed to unmarshal %s", string(b))
}
if f1.And.ObjectSizeLessThan != f2.And.ObjectSizeLessThan {
t.Fatalf("Expected %v but got %v", f1.And.ObjectSizeLessThan, f2.And.ObjectSizeLessThan)
}
if f1.And.ObjectSizeGreaterThan != f2.And.ObjectSizeGreaterThan {
t.Fatalf("Expected %v but got %v", f1.And.ObjectSizeGreaterThan, f2.And.ObjectSizeGreaterThan)
}
fiGt := Filter{
ObjectSizeGreaterThan: 1 * humanize.MiByte,
}
fiLt := Filter{
ObjectSizeLessThan: 100 * humanize.MiByte,
}
fiLtAndGt := Filter{
And: And{
ObjectSizeGreaterThan: 1 * humanize.MiByte,
ObjectSizeLessThan: 100 * humanize.MiByte,
},
}
tests := []struct {
filter Filter
objSize int64
want bool
}{
{
filter: fiLt,
objSize: 101 * humanize.MiByte,
want: false,
},
{
filter: fiLt,
objSize: 99 * humanize.MiByte,
want: true,
},
{
filter: fiGt,
objSize: 1*humanize.MiByte - 1,
want: false,
},
{
filter: fiGt,
objSize: 1*humanize.MiByte + 1,
want: true,
},
{
filter: fiLtAndGt,
objSize: 1*humanize.MiByte - 1,
want: false,
},
{
filter: fiLtAndGt,
objSize: 2 * humanize.MiByte,
want: true,
},
{
filter: fiLtAndGt,
objSize: 100*humanize.MiByte + 1,
want: false,
},
}
for i, test := range tests {
t.Run(fmt.Sprintf("Test %d", i+1), func(t *testing.T) {
if got := test.filter.BySize(test.objSize); got != test.want {
t.Fatalf("Expected %v but got %v", test.want, got)
}
})
}
}
func TestTestTags(t *testing.T) {
noTags := Filter{
set: true,
And: And{
Tags: []Tag{},
},
andSet: true,
}
oneTag := Filter{
set: true,
And: And{
Tags: []Tag{{Key: "FOO", Value: "1"}},
},
andSet: true,
}
twoTags := Filter{
set: true,
And: And{
Tags: []Tag{{Key: "FOO", Value: "1"}, {Key: "BAR", Value: "2"}},
},
andSet: true,
}
tests := []struct {
filter Filter
userTags string
want bool
}{
{
filter: noTags,
userTags: "",
want: true,
},
{
filter: noTags,
userTags: "A=3",
want: true,
},
{
filter: oneTag,
userTags: "A=3",
want: false,
},
{
filter: oneTag,
userTags: "FOO=1",
want: true,
},
{
filter: oneTag,
userTags: "A=B&FOO=1",
want: true,
},
{
filter: twoTags,
userTags: "",
want: false,
},
{
filter: twoTags,
userTags: "FOO=1",
want: false,
},
{
filter: twoTags,
userTags: "BAR=2",
want: false,
},
{
filter: twoTags,
userTags: "FOO=2&BAR=2",
want: false,
},
{
filter: twoTags,
userTags: "F=1&B=2",
want: false,
},
{
filter: twoTags,
userTags: "FOO=1&BAR=2",
want: true,
},
{
filter: twoTags,
userTags: "BAR=2&FOO=1",
want: true,
},
}
for i, test := range tests {
t.Run(fmt.Sprintf("Test %d", i+1), func(t *testing.T) {
if got := test.filter.TestTags(test.userTags); got != test.want {
t.Errorf("Expected %v but got %v", test.want, got)
}
})
}
}

View File

@@ -0,0 +1,570 @@
// Copyright (c) 2015-2024 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package lifecycle
import (
"encoding/xml"
"fmt"
"io"
"net/http"
"slices"
"strings"
"time"
"github.com/google/uuid"
"github.com/minio/minio/internal/bucket/object/lock"
"github.com/minio/minio/internal/bucket/replication"
xhttp "github.com/minio/minio/internal/http"
)
var (
errLifecycleTooManyRules = Errorf("Lifecycle configuration allows a maximum of 1000 rules")
errLifecycleNoRule = Errorf("Lifecycle configuration should have at least one rule")
errLifecycleDuplicateID = Errorf("Rule ID must be unique. Found same ID for more than one rule")
errXMLNotWellFormed = Errorf("The XML you provided was not well-formed or did not validate against our published schema")
errLifecycleBucketLocked = Errorf("ExpiredObjectAllVersions element and DelMarkerExpiration action cannot be used on an object locked bucket")
)
const (
// TransitionComplete marks completed transition
TransitionComplete = "complete"
// TransitionPending - transition is yet to be attempted
TransitionPending = "pending"
)
// Action represents a delete action or other transition
// actions that will be implemented later.
type Action int
//go:generate stringer -type Action $GOFILE
const (
// NoneAction means no action required after evaluating lifecycle rules
NoneAction Action = iota
// DeleteAction means the object needs to be removed after evaluating lifecycle rules
DeleteAction
// DeleteVersionAction deletes a particular version
DeleteVersionAction
// TransitionAction transitions a particular object after evaluating lifecycle transition rules
TransitionAction
// TransitionVersionAction transitions a particular object version after evaluating lifecycle transition rules
TransitionVersionAction
// DeleteRestoredAction means the temporarily restored object needs to be removed after evaluating lifecycle rules
DeleteRestoredAction
// DeleteRestoredVersionAction deletes a particular version that was temporarily restored
DeleteRestoredVersionAction
// DeleteAllVersionsAction deletes all versions when an object expires
DeleteAllVersionsAction
// DelMarkerDeleteAllVersionsAction deletes all versions when an object with delete marker as latest version expires
DelMarkerDeleteAllVersionsAction
// ActionCount must be the last action and shouldn't be used as a regular action.
ActionCount
)
// DeleteRestored - Returns true if action demands delete on restored objects
func (a Action) DeleteRestored() bool {
return a == DeleteRestoredAction || a == DeleteRestoredVersionAction
}
// DeleteVersioned - Returns true if action demands delete on a versioned object
func (a Action) DeleteVersioned() bool {
return a == DeleteVersionAction || a == DeleteRestoredVersionAction
}
// DeleteAll - Returns true if the action demands deleting all versions of an object
func (a Action) DeleteAll() bool {
return a == DeleteAllVersionsAction || a == DelMarkerDeleteAllVersionsAction
}
// Delete - Returns true if action demands delete on all objects (including restored)
func (a Action) Delete() bool {
if a.DeleteRestored() {
return true
}
return a == DeleteVersionAction || a == DeleteAction || a == DeleteAllVersionsAction || a == DelMarkerDeleteAllVersionsAction
}
// Lifecycle - Configuration for bucket lifecycle.
type Lifecycle struct {
XMLName xml.Name `xml:"LifecycleConfiguration"`
Rules []Rule `xml:"Rule"`
ExpiryUpdatedAt *time.Time `xml:"ExpiryUpdatedAt,omitempty"`
}
// HasTransition returns 'true' if lifecycle document has Transition enabled.
func (lc Lifecycle) HasTransition() bool {
for _, rule := range lc.Rules {
if rule.Transition.IsEnabled() {
return true
}
}
return false
}
// HasExpiry returns 'true' if lifecycle document has Expiry enabled.
func (lc Lifecycle) HasExpiry() bool {
for _, rule := range lc.Rules {
if !rule.Expiration.IsNull() || !rule.NoncurrentVersionExpiration.IsNull() {
return true
}
}
return false
}
// UnmarshalXML - decodes XML data.
func (lc *Lifecycle) UnmarshalXML(d *xml.Decoder, start xml.StartElement) (err error) {
switch start.Name.Local {
case "LifecycleConfiguration", "BucketLifecycleConfiguration":
default:
return xml.UnmarshalError(fmt.Sprintf("expected element type <LifecycleConfiguration>/<BucketLifecycleConfiguration> but have <%s>",
start.Name.Local))
}
for {
// Read tokens from the XML document in a stream.
t, err := d.Token()
if err != nil {
if err == io.EOF {
break
}
return err
}
if se, ok := t.(xml.StartElement); ok {
switch se.Name.Local {
case "Rule":
var r Rule
if err = d.DecodeElement(&r, &se); err != nil {
return err
}
lc.Rules = append(lc.Rules, r)
case "ExpiryUpdatedAt":
var t time.Time
if err = d.DecodeElement(&t, &start); err != nil {
return err
}
lc.ExpiryUpdatedAt = &t
default:
return xml.UnmarshalError(fmt.Sprintf("expected element type <Rule> but have <%s>", se.Name.Local))
}
}
}
return nil
}
// HasActiveRules - returns whether lc has active rules at any level below or at prefix.
func (lc Lifecycle) HasActiveRules(prefix string) bool {
if len(lc.Rules) == 0 {
return false
}
for _, rule := range lc.Rules {
if rule.Status == Disabled {
continue
}
if len(prefix) > 0 && len(rule.GetPrefix()) > 0 {
// we can skip this rule if it doesn't match the tested
// prefix.
if !strings.HasPrefix(prefix, rule.GetPrefix()) && !strings.HasPrefix(rule.GetPrefix(), prefix) {
continue
}
}
if rule.NoncurrentVersionExpiration.NoncurrentDays > 0 {
return true
}
if rule.NoncurrentVersionExpiration.NewerNoncurrentVersions > 0 {
return true
}
if !rule.NoncurrentVersionTransition.IsNull() {
return true
}
if !rule.Expiration.IsDateNull() && rule.Expiration.Date.Before(time.Now().UTC()) {
return true
}
if !rule.Expiration.IsDaysNull() {
return true
}
if rule.Expiration.DeleteMarker.val {
return true
}
if !rule.Transition.IsDateNull() && rule.Transition.Date.Before(time.Now().UTC()) {
return true
}
if !rule.Transition.IsNull() { // this allows for Transition.Days to be zero.
return true
}
}
return false
}
// ParseLifecycleConfigWithID - parses for a Lifecycle config and assigns
// unique id to rules with empty ID.
func ParseLifecycleConfigWithID(r io.Reader) (*Lifecycle, error) {
var lc Lifecycle
if err := xml.NewDecoder(r).Decode(&lc); err != nil {
return nil, err
}
// assign a unique id for rules with empty ID
for i := range lc.Rules {
if lc.Rules[i].ID == "" {
lc.Rules[i].ID = uuid.New().String()
}
}
return &lc, nil
}
// ParseLifecycleConfig - parses data in given reader to Lifecycle.
func ParseLifecycleConfig(reader io.Reader) (*Lifecycle, error) {
var lc Lifecycle
if err := xml.NewDecoder(reader).Decode(&lc); err != nil {
return nil, err
}
return &lc, nil
}
// Validate - validates the lifecycle configuration
func (lc Lifecycle) Validate(lr lock.Retention) error {
// Lifecycle config can't have more than 1000 rules
if len(lc.Rules) > 1000 {
return errLifecycleTooManyRules
}
// Lifecycle config should have at least one rule
if len(lc.Rules) == 0 {
return errLifecycleNoRule
}
// Validate all the rules in the lifecycle config
for _, r := range lc.Rules {
if err := r.Validate(); err != nil {
return err
}
if lr.LockEnabled && (r.Expiration.DeleteAll.val || !r.DelMarkerExpiration.Empty()) {
return errLifecycleBucketLocked
}
}
// Make sure Rule ID is unique
for i := range lc.Rules {
if i == len(lc.Rules)-1 {
break
}
otherRules := lc.Rules[i+1:]
for _, otherRule := range otherRules {
if lc.Rules[i].ID == otherRule.ID {
return errLifecycleDuplicateID
}
}
}
return nil
}
// FilterRules returns the rules filtered by the status, prefix and tags
func (lc Lifecycle) FilterRules(obj ObjectOpts) []Rule {
if obj.Name == "" {
return nil
}
var rules []Rule
for _, rule := range lc.Rules {
if rule.Status == Disabled {
continue
}
if !strings.HasPrefix(obj.Name, rule.GetPrefix()) {
continue
}
if !rule.Filter.TestTags(obj.UserTags) {
continue
}
if !obj.DeleteMarker && !rule.Filter.BySize(obj.Size) {
continue
}
rules = append(rules, rule)
}
return rules
}
// ObjectOpts provides information to deduce the lifecycle actions
// which can be triggered on the resultant object.
type ObjectOpts struct {
Name string
UserTags string
ModTime time.Time
Size int64
VersionID string
IsLatest bool
DeleteMarker bool
NumVersions int
SuccessorModTime time.Time
TransitionStatus string
RestoreOngoing bool
RestoreExpires time.Time
// to determine if object is locked due to retention
UserDefined map[string]string
VersionPurgeStatus replication.VersionPurgeStatusType
ReplicationStatus replication.StatusType
}
// ExpiredObjectDeleteMarker returns true if an object version referred to by o
// is the only version remaining and is a delete marker. It returns false
// otherwise.
func (o ObjectOpts) ExpiredObjectDeleteMarker() bool {
return o.DeleteMarker && o.NumVersions == 1
}
// Event contains a lifecycle action with associated info
type Event struct {
Action Action
RuleID string
Due time.Time
NoncurrentDays int
NewerNoncurrentVersions int
StorageClass string
}
// Eval returns the lifecycle event applicable now.
func (lc Lifecycle) Eval(obj ObjectOpts) Event {
return lc.eval(obj, time.Now().UTC(), 0)
}
// eval returns the lifecycle event applicable at the given now. If now is the
// zero value of time.Time, it returns the upcoming lifecycle event.
func (lc Lifecycle) eval(obj ObjectOpts, now time.Time, remainingVersions int) Event {
var events []Event
if obj.ModTime.IsZero() {
return Event{}
}
// Handle expiry of restored object; NB Restored Objects have expiry set on
// them as part of RestoreObject API. They aren't governed by lifecycle
// rules.
if !obj.RestoreExpires.IsZero() && now.After(obj.RestoreExpires) {
action := DeleteRestoredAction
if !obj.IsLatest {
action = DeleteRestoredVersionAction
}
events = append(events, Event{
Action: action,
Due: now,
})
}
for _, rule := range lc.FilterRules(obj) {
if obj.ExpiredObjectDeleteMarker() {
if rule.Expiration.DeleteMarker.val {
// Indicates whether MinIO will remove a delete marker with no noncurrent versions.
// Only latest marker is removed. If set to true, the delete marker will be expired;
// if set to false the policy takes no action. This cannot be specified with Days or
// Date in a Lifecycle Expiration Policy.
events = append(events, Event{
Action: DeleteVersionAction,
RuleID: rule.ID,
Due: now,
})
// No other conflicting actions apply to an expired object delete marker
break
}
if !rule.Expiration.IsDaysNull() {
// Specifying the Days tag will automatically perform ExpiredObjectDeleteMarker cleanup
// once delete markers are old enough to satisfy the age criteria.
// https://docs.aws.amazon.com/AmazonS3/latest/userguide/lifecycle-configuration-examples.html
if expectedExpiry := ExpectedExpiryTime(obj.ModTime, int(rule.Expiration.Days)); now.IsZero() || now.After(expectedExpiry) {
events = append(events, Event{
Action: DeleteVersionAction,
RuleID: rule.ID,
Due: expectedExpiry,
})
// No other conflicting actions apply to an expired object delete marker
break
}
}
}
// DelMarkerExpiration
if obj.IsLatest && obj.DeleteMarker && !rule.DelMarkerExpiration.Empty() {
if due, ok := rule.DelMarkerExpiration.NextDue(obj); ok && (now.IsZero() || now.After(due)) {
events = append(events, Event{
Action: DelMarkerDeleteAllVersionsAction,
RuleID: rule.ID,
Due: due,
})
}
// No other conflicting actions in this rule can apply to an object with current version as DEL marker
// Note: There could be other rules with earlier expiration which need to be considered.
// See TestDelMarkerExpiration
continue
}
// NoncurrentVersionExpiration
if !obj.IsLatest && rule.NoncurrentVersionExpiration.set {
var (
retainedEnough bool
oldEnough bool
)
if rule.NoncurrentVersionExpiration.NewerNoncurrentVersions == 0 || remainingVersions >= rule.NoncurrentVersionExpiration.NewerNoncurrentVersions {
retainedEnough = true
}
expectedExpiry := ExpectedExpiryTime(obj.SuccessorModTime, int(rule.NoncurrentVersionExpiration.NoncurrentDays))
if now.IsZero() || now.After(expectedExpiry) {
oldEnough = true
}
// > For the deletion to occur, both the <NoncurrentDays> and the <NewerNoncurrentVersions> values must be exceeded.
// ref: https://docs.aws.amazon.com/AmazonS3/latest/dev/intro-lifecycle-rules.html#intro-lifecycle-rules-actions
if retainedEnough && oldEnough {
events = append(events, Event{
Action: DeleteVersionAction,
RuleID: rule.ID,
Due: expectedExpiry,
})
}
}
if !obj.IsLatest && !rule.NoncurrentVersionTransition.IsNull() {
if !obj.DeleteMarker && obj.TransitionStatus != TransitionComplete {
// Non current versions should be transitioned if their age exceeds non current days configuration
// https://docs.aws.amazon.com/AmazonS3/latest/dev/intro-lifecycle-rules.html#intro-lifecycle-rules-actions
if due, ok := rule.NoncurrentVersionTransition.NextDue(obj); ok && (now.IsZero() || now.After(due)) {
events = append(events, Event{
Action: TransitionVersionAction,
RuleID: rule.ID,
Due: due,
StorageClass: rule.NoncurrentVersionTransition.StorageClass,
})
}
}
}
// Remove the object or simply add a delete marker (once) in a versioned bucket
if obj.IsLatest && !obj.DeleteMarker {
switch {
case !rule.Expiration.IsDateNull():
if now.IsZero() || now.After(rule.Expiration.Date.Time) {
events = append(events, Event{
Action: DeleteAction,
RuleID: rule.ID,
Due: rule.Expiration.Date.Time,
})
}
case !rule.Expiration.IsDaysNull():
if expectedExpiry := ExpectedExpiryTime(obj.ModTime, int(rule.Expiration.Days)); now.IsZero() || now.After(expectedExpiry) {
event := Event{
Action: DeleteAction,
RuleID: rule.ID,
Due: expectedExpiry,
}
if rule.Expiration.DeleteAll.val {
// Expires all versions of this object once the latest object is old enough.
// This is a MinIO only extension.
event.Action = DeleteAllVersionsAction
}
events = append(events, event)
}
}
if obj.TransitionStatus != TransitionComplete {
if due, ok := rule.Transition.NextDue(obj); ok && (now.IsZero() || now.After(due)) {
events = append(events, Event{
Action: TransitionAction,
RuleID: rule.ID,
Due: due,
StorageClass: rule.Transition.StorageClass,
})
}
}
}
}
if len(events) > 0 {
slices.SortFunc(events, func(a, b Event) int {
// Prefer Expiration over Transition for both current
// and noncurrent versions when,
// - now is past the expected time to action
// - expected time to action is the same for both actions
if now.After(a.Due) && now.After(b.Due) || a.Due.Equal(b.Due) {
switch a.Action {
case DeleteAllVersionsAction, DelMarkerDeleteAllVersionsAction,
DeleteAction, DeleteVersionAction:
return -1
}
switch b.Action {
case DeleteAllVersionsAction, DelMarkerDeleteAllVersionsAction,
DeleteAction, DeleteVersionAction:
return 1
}
return -1
}
// Prefer earlier occurring event
if a.Due.Before(b.Due) {
return -1
}
return 1
})
return events[0]
}
return Event{
Action: NoneAction,
}
}
// ExpectedExpiryTime calculates the expiry, transition or restore date/time based on a object modtime.
// The expected transition or restore time is always a midnight time following the object
// modification time plus the number of transition/restore days.
//
// e.g. If the object modtime is `Thu May 21 13:42:50 GMT 2020` and the object should
// transition in 1 day, then the expected transition time is `Fri, 23 May 2020 00:00:00 GMT`
func ExpectedExpiryTime(modTime time.Time, days int) time.Time {
if days == 0 {
return modTime
}
t := modTime.UTC().Add(time.Duration(days+1) * 24 * time.Hour)
return t.Truncate(24 * time.Hour)
}
// SetPredictionHeaders sets time to expiry and transition headers on w for a
// given obj.
func (lc Lifecycle) SetPredictionHeaders(w http.ResponseWriter, obj ObjectOpts) {
event := lc.eval(obj, time.Time{}, 0)
switch event.Action {
case DeleteAction, DeleteVersionAction, DeleteAllVersionsAction, DelMarkerDeleteAllVersionsAction:
w.Header()[xhttp.AmzExpiration] = []string{
fmt.Sprintf(`expiry-date="%s", rule-id="%s"`, event.Due.Format(http.TimeFormat), event.RuleID),
}
case TransitionAction, TransitionVersionAction:
w.Header()[xhttp.MinIOTransition] = []string{
fmt.Sprintf(`transition-date="%s", rule-id="%s"`, event.Due.Format(http.TimeFormat), event.RuleID),
}
}
}
// NoncurrentVersionsExpirationLimit returns the number of noncurrent versions
// to be retained from the first applicable rule per S3 behavior.
func (lc Lifecycle) NoncurrentVersionsExpirationLimit(obj ObjectOpts) Event {
for _, rule := range lc.FilterRules(obj) {
if rule.NoncurrentVersionExpiration.NewerNoncurrentVersions == 0 {
continue
}
return Event{
Action: DeleteVersionAction,
RuleID: rule.ID,
NoncurrentDays: int(rule.NoncurrentVersionExpiration.NoncurrentDays),
NewerNoncurrentVersions: rule.NoncurrentVersionExpiration.NewerNoncurrentVersions,
}
}
return Event{}
}

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,156 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package lifecycle
import (
"encoding/xml"
"time"
)
// NoncurrentVersionExpiration - an action for lifecycle configuration rule.
type NoncurrentVersionExpiration struct {
XMLName xml.Name `xml:"NoncurrentVersionExpiration"`
NoncurrentDays ExpirationDays `xml:"NoncurrentDays,omitempty"`
NewerNoncurrentVersions int `xml:"NewerNoncurrentVersions,omitempty"`
set bool
}
// MarshalXML if non-current days not set to non zero value
func (n NoncurrentVersionExpiration) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
if n.IsNull() {
return nil
}
type noncurrentVersionExpirationWrapper NoncurrentVersionExpiration
return e.EncodeElement(noncurrentVersionExpirationWrapper(n), start)
}
// UnmarshalXML decodes NoncurrentVersionExpiration
func (n *NoncurrentVersionExpiration) UnmarshalXML(d *xml.Decoder, startElement xml.StartElement) error {
// To handle xml with MaxNoncurrentVersions from older MinIO releases.
// note: only one of MaxNoncurrentVersions or NewerNoncurrentVersions would be present.
type noncurrentExpiration struct {
XMLName xml.Name `xml:"NoncurrentVersionExpiration"`
NoncurrentDays ExpirationDays `xml:"NoncurrentDays,omitempty"`
NewerNoncurrentVersions int `xml:"NewerNoncurrentVersions,omitempty"`
MaxNoncurrentVersions int `xml:"MaxNoncurrentVersions,omitempty"`
}
var val noncurrentExpiration
err := d.DecodeElement(&val, &startElement)
if err != nil {
return err
}
if val.MaxNoncurrentVersions > 0 {
val.NewerNoncurrentVersions = val.MaxNoncurrentVersions
}
*n = NoncurrentVersionExpiration{
XMLName: val.XMLName,
NoncurrentDays: val.NoncurrentDays,
NewerNoncurrentVersions: val.NewerNoncurrentVersions,
}
n.set = true
return nil
}
// IsNull returns if both NoncurrentDays and NoncurrentVersions are empty
func (n NoncurrentVersionExpiration) IsNull() bool {
return n.IsDaysNull() && n.NewerNoncurrentVersions == 0
}
// IsDaysNull returns true if days field is null
func (n NoncurrentVersionExpiration) IsDaysNull() bool {
return n.NoncurrentDays == ExpirationDays(0)
}
// Validate returns an error with wrong value
func (n NoncurrentVersionExpiration) Validate() error {
if !n.set {
return nil
}
val := int(n.NoncurrentDays)
switch {
case val == 0 && n.NewerNoncurrentVersions == 0:
// both fields can't be zero
return errXMLNotWellFormed
case val < 0, n.NewerNoncurrentVersions < 0:
// negative values are not supported
return errXMLNotWellFormed
}
return nil
}
// NoncurrentVersionTransition - an action for lifecycle configuration rule.
type NoncurrentVersionTransition struct {
NoncurrentDays TransitionDays `xml:"NoncurrentDays"`
StorageClass string `xml:"StorageClass"`
set bool
}
// MarshalXML is extended to leave out
// <NoncurrentVersionTransition></NoncurrentVersionTransition> tags
func (n NoncurrentVersionTransition) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
if n.IsNull() {
return nil
}
type noncurrentVersionTransitionWrapper NoncurrentVersionTransition
return e.EncodeElement(noncurrentVersionTransitionWrapper(n), start)
}
// UnmarshalXML decodes NoncurrentVersionExpiration
func (n *NoncurrentVersionTransition) UnmarshalXML(d *xml.Decoder, startElement xml.StartElement) error {
type noncurrentVersionTransitionWrapper NoncurrentVersionTransition
var val noncurrentVersionTransitionWrapper
err := d.DecodeElement(&val, &startElement)
if err != nil {
return err
}
*n = NoncurrentVersionTransition(val)
n.set = true
return nil
}
// IsNull returns true if NoncurrentTransition doesn't refer to any storage-class.
// Note: It supports immediate transition, i.e zero noncurrent days.
func (n NoncurrentVersionTransition) IsNull() bool {
return n.StorageClass == ""
}
// Validate returns an error with wrong value
func (n NoncurrentVersionTransition) Validate() error {
if !n.set {
return nil
}
if n.StorageClass == "" {
return errXMLNotWellFormed
}
return nil
}
// NextDue returns upcoming NoncurrentVersionTransition date for obj if
// applicable, returns false otherwise.
func (n NoncurrentVersionTransition) NextDue(obj ObjectOpts) (time.Time, bool) {
if obj.IsLatest || n.StorageClass == "" {
return time.Time{}, false
}
// Days == 0 indicates immediate tiering, i.e object is eligible for tiering since it became noncurrent.
if n.NoncurrentDays == 0 {
return obj.SuccessorModTime, true
}
return ExpectedExpiryTime(obj.SuccessorModTime, int(n.NoncurrentDays)), true
}

View File

@@ -0,0 +1,82 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package lifecycle
import "testing"
func Test_NoncurrentVersionsExpiration_Validation(t *testing.T) {
testcases := []struct {
n NoncurrentVersionExpiration
err error
}{
{
n: NoncurrentVersionExpiration{
NoncurrentDays: 0,
NewerNoncurrentVersions: 0,
set: true,
},
err: errXMLNotWellFormed,
},
{
n: NoncurrentVersionExpiration{
NoncurrentDays: 90,
NewerNoncurrentVersions: 0,
set: true,
},
err: nil,
},
{
n: NoncurrentVersionExpiration{
NoncurrentDays: 90,
NewerNoncurrentVersions: 2,
set: true,
},
err: nil,
},
{
n: NoncurrentVersionExpiration{
NoncurrentDays: -1,
set: true,
},
err: errXMLNotWellFormed,
},
{
n: NoncurrentVersionExpiration{
NoncurrentDays: 90,
NewerNoncurrentVersions: -2,
set: true,
},
err: errXMLNotWellFormed,
},
// MinIO extension: supports zero NoncurrentDays when NewerNoncurrentVersions > 0
{
n: NoncurrentVersionExpiration{
NoncurrentDays: 0,
NewerNoncurrentVersions: 5,
set: true,
},
err: nil,
},
}
for i, tc := range testcases {
if got := tc.n.Validate(); got != tc.err {
t.Fatalf("%d: expected %v but got %v", i+1, tc.err, got)
}
}
}

View File

@@ -0,0 +1,52 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package lifecycle
import (
"encoding/xml"
)
// Prefix holds the prefix xml tag in <Rule> and <Filter>
type Prefix struct {
string
set bool
Unused struct{} // Needed for GOB compatibility
}
// UnmarshalXML - decodes XML data.
func (p *Prefix) UnmarshalXML(d *xml.Decoder, start xml.StartElement) (err error) {
var s string
if err = d.DecodeElement(&s, &start); err != nil {
return err
}
*p = Prefix{string: s, set: true}
return nil
}
// MarshalXML - decodes XML data.
func (p Prefix) MarshalXML(e *xml.Encoder, startElement xml.StartElement) error {
if !p.set {
return nil
}
return e.EncodeElement(p.string, startElement)
}
// String returns the prefix string
func (p Prefix) String() string {
return p.string
}

View File

@@ -0,0 +1,194 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package lifecycle
import (
"bytes"
"encoding/xml"
)
// Status represents lifecycle configuration status
type Status string
// Supported status types
const (
Enabled Status = "Enabled"
Disabled Status = "Disabled"
)
// Rule - a rule for lifecycle configuration.
type Rule struct {
XMLName xml.Name `xml:"Rule"`
ID string `xml:"ID,omitempty"`
Status Status `xml:"Status"`
Filter Filter `xml:"Filter,omitempty"`
Prefix Prefix `xml:"Prefix,omitempty"`
Expiration Expiration `xml:"Expiration,omitempty"`
Transition Transition `xml:"Transition,omitempty"`
DelMarkerExpiration DelMarkerExpiration `xml:"DelMarkerExpiration,omitempty"`
// FIXME: add a type to catch unsupported AbortIncompleteMultipartUpload AbortIncompleteMultipartUpload `xml:"AbortIncompleteMultipartUpload,omitempty"`
NoncurrentVersionExpiration NoncurrentVersionExpiration `xml:"NoncurrentVersionExpiration,omitempty"`
NoncurrentVersionTransition NoncurrentVersionTransition `xml:"NoncurrentVersionTransition,omitempty"`
}
var (
errInvalidRuleID = Errorf("ID length is limited to 255 characters")
errEmptyRuleStatus = Errorf("Status should not be empty")
errInvalidRuleStatus = Errorf("Status must be set to either Enabled or Disabled")
errInvalidRuleDelMarkerExpiration = Errorf("Rule with DelMarkerExpiration cannot have tags based filtering")
)
// validateID - checks if ID is valid or not.
func (r Rule) validateID() error {
if len(r.ID) > 255 {
return errInvalidRuleID
}
return nil
}
// validateStatus - checks if status is valid or not.
func (r Rule) validateStatus() error {
// Status can't be empty
if len(r.Status) == 0 {
return errEmptyRuleStatus
}
// Status must be one of Enabled or Disabled
if r.Status != Enabled && r.Status != Disabled {
return errInvalidRuleStatus
}
return nil
}
func (r Rule) validateExpiration() error {
return r.Expiration.Validate()
}
func (r Rule) validateNoncurrentExpiration() error {
return r.NoncurrentVersionExpiration.Validate()
}
func (r Rule) validatePrefixAndFilter() error {
// In the now deprecated PutBucketLifecycle API, Rule had a mandatory Prefix element and there existed no Filter field.
// See https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketLifecycle.html
// In the newer PutBucketLifecycleConfiguration API, Rule has a prefix field that is deprecated, and there exists an optional
// Filter field, and within it, an optional Prefix field.
// See https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketLifecycleConfiguration.html
// A valid rule could be a pre-existing one created using the now deprecated PutBucketLifecycle.
// Or, a valid rule could also be either a pre-existing or a new rule that is created using PutBucketLifecycleConfiguration.
// Prefix validation below may check that either Rule.Prefix or Rule.Filter.Prefix exist but not both.
// Here, we assume the pre-existing rule created using PutBucketLifecycle API is already valid and won't fail the validation if Rule.Prefix is empty.
if r.Prefix.set && !r.Filter.IsEmpty() && r.Filter.Prefix.set {
return errXMLNotWellFormed
}
if r.Filter.set {
return r.Filter.Validate()
}
return nil
}
func (r Rule) validateTransition() error {
return r.Transition.Validate()
}
func (r Rule) validateNoncurrentTransition() error {
return r.NoncurrentVersionTransition.Validate()
}
// GetPrefix - a rule can either have prefix under <rule></rule>, <filter></filter>
// or under <filter><and></and></filter>. This method returns the prefix from the
// location where it is available.
func (r Rule) GetPrefix() string {
if p := r.Prefix.String(); p != "" {
return p
}
if p := r.Filter.Prefix.String(); p != "" {
return p
}
if p := r.Filter.And.Prefix.String(); p != "" {
return p
}
return ""
}
// Tags - a rule can either have tag under <filter></filter> or under
// <filter><and></and></filter>. This method returns all the tags from the
// rule in the format tag1=value1&tag2=value2
func (r Rule) Tags() string {
if !r.Filter.Tag.IsEmpty() {
return r.Filter.Tag.String()
}
if len(r.Filter.And.Tags) != 0 {
var buf bytes.Buffer
for _, t := range r.Filter.And.Tags {
if buf.Len() > 0 {
buf.WriteString("&")
}
buf.WriteString(t.String())
}
return buf.String()
}
return ""
}
// Validate - validates the rule element
func (r Rule) Validate() error {
if err := r.validateID(); err != nil {
return err
}
if err := r.validateStatus(); err != nil {
return err
}
if err := r.validateExpiration(); err != nil {
return err
}
if err := r.validateNoncurrentExpiration(); err != nil {
return err
}
if err := r.validatePrefixAndFilter(); err != nil {
return err
}
if err := r.validateTransition(); err != nil {
return err
}
if err := r.validateNoncurrentTransition(); err != nil {
return err
}
if (!r.Filter.Tag.IsEmpty() || len(r.Filter.And.Tags) != 0) && !r.DelMarkerExpiration.Empty() {
return errInvalidRuleDelMarkerExpiration
}
if !r.Expiration.set && !r.Transition.set && !r.NoncurrentVersionExpiration.set && !r.NoncurrentVersionTransition.set && r.DelMarkerExpiration.Empty() {
return errXMLNotWellFormed
}
return nil
}
// CloneNonTransition - returns a clone of the object containing non transition rules
func (r Rule) CloneNonTransition() Rule {
return Rule{
XMLName: r.XMLName,
ID: r.ID,
Status: r.Status,
Filter: r.Filter,
Prefix: r.Prefix,
Expiration: r.Expiration,
NoncurrentVersionExpiration: r.NoncurrentVersionExpiration,
}
}

View File

@@ -0,0 +1,148 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package lifecycle
import (
"encoding/xml"
"fmt"
"testing"
)
// TestInvalidRules checks if Rule xml with invalid elements returns
// appropriate errors on validation
func TestInvalidRules(t *testing.T) {
invalidTestCases := []struct {
inputXML string
expectedErr error
}{
{ // Rule with ID longer than 255 characters
inputXML: ` <Rule>
<ID> babababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababab </ID>
</Rule>`,
expectedErr: errInvalidRuleID,
},
{ // Rule with empty ID
inputXML: `<Rule>
<ID></ID>
<Filter><Prefix></Prefix></Filter>
<Expiration>
<Days>365</Days>
</Expiration>
<Status>Enabled</Status>
</Rule>`,
expectedErr: nil,
},
{ // Rule with empty status
inputXML: ` <Rule>
<ID>rule with empty status</ID>
<Status></Status>
</Rule>`,
expectedErr: errEmptyRuleStatus,
},
{ // Rule with invalid status
inputXML: ` <Rule>
<ID>rule with invalid status</ID>
<Status>OK</Status>
</Rule>`,
expectedErr: errInvalidRuleStatus,
},
{ // Rule with negative values for ObjectSizeLessThan
inputXML: `<Rule>
<ID>negative-obj-size-less-than</ID>
<Filter><ObjectSizeLessThan>-1</ObjectSizeLessThan></Filter>
<Expiration>
<Days>365</Days>
</Expiration>
<Status>Enabled</Status>
</Rule>`,
expectedErr: errXMLNotWellFormed,
},
{ // Rule with negative values for And>ObjectSizeLessThan
inputXML: `<Rule>
<ID>negative-and-obj-size-less-than</ID>
<Filter><And><ObjectSizeLessThan>-1</ObjectSizeLessThan></And></Filter>
<Expiration>
<Days>365</Days>
</Expiration>
<Status>Enabled</Status>
</Rule>`,
expectedErr: errXMLNotWellFormed,
},
{ // Rule with negative values for ObjectSizeGreaterThan
inputXML: `<Rule>
<ID>negative-obj-size-greater-than</ID>
<Filter><ObjectSizeGreaterThan>-1</ObjectSizeGreaterThan></Filter>
<Expiration>
<Days>365</Days>
</Expiration>
<Status>Enabled</Status>
</Rule>`,
expectedErr: errXMLNotWellFormed,
},
{ // Rule with negative values for And>ObjectSizeGreaterThan
inputXML: `<Rule>
<ID>negative-and-obj-size-greater-than</ID>
<Filter><And><ObjectSizeGreaterThan>-1</ObjectSizeGreaterThan></And></Filter>
<Expiration>
<Days>365</Days>
</Expiration>
<Status>Enabled</Status>
</Rule>`,
expectedErr: errXMLNotWellFormed,
},
{
inputXML: `<Rule>
<ID>Rule with a tag and DelMarkerExpiration</ID>
<Filter><Tag><Key>k1</Key><Value>v1</Value></Tag></Filter>
<DelMarkerExpiration>
<Days>365</Days>
</DelMarkerExpiration>
<Status>Enabled</Status>
</Rule>`,
expectedErr: errInvalidRuleDelMarkerExpiration,
},
{
inputXML: `<Rule>
<ID>Rule with multiple tags and DelMarkerExpiration</ID>
<Filter><And>
<Tag><Key>k1</Key><Value>v1</Value></Tag>
<Tag><Key>k2</Key><Value>v2</Value></Tag>
</And></Filter>
<DelMarkerExpiration>
<Days>365</Days>
</DelMarkerExpiration>
<Status>Enabled</Status>
</Rule>`,
expectedErr: errInvalidRuleDelMarkerExpiration,
},
}
for i, tc := range invalidTestCases {
t.Run(fmt.Sprintf("Test %d", i+1), func(t *testing.T) {
var rule Rule
err := xml.Unmarshal([]byte(tc.inputXML), &rule)
if err != nil {
t.Fatal(err)
}
if err := rule.Validate(); err != tc.expectedErr {
t.Fatalf("%d: Expected %v but got %v", i+1, tc.expectedErr, err)
}
})
}
}

View File

@@ -0,0 +1,101 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package lifecycle
import (
"encoding/xml"
"io"
"unicode/utf8"
)
// Tag - a tag for a lifecycle configuration Rule filter.
type Tag struct {
XMLName xml.Name `xml:"Tag"`
Key string `xml:"Key,omitempty"`
Value string `xml:"Value,omitempty"`
}
var (
errInvalidTagKey = Errorf("The TagKey you have provided is invalid")
errInvalidTagValue = Errorf("The TagValue you have provided is invalid")
errDuplicatedXMLTag = Errorf("duplicated XML Tag")
errUnknownXMLTag = Errorf("unknown XML Tag")
)
// UnmarshalXML - decodes XML data.
func (tag *Tag) UnmarshalXML(d *xml.Decoder, start xml.StartElement) (err error) {
var keyAlreadyParsed, valueAlreadyParsed bool
for {
// Read tokens from the XML document in a stream.
t, err := d.Token()
if err != nil {
if err == io.EOF {
break
}
return err
}
if se, ok := t.(xml.StartElement); ok {
var s string
if err = d.DecodeElement(&s, &se); err != nil {
return err
}
switch se.Name.Local {
case "Key":
if keyAlreadyParsed {
return errDuplicatedXMLTag
}
tag.Key = s
keyAlreadyParsed = true
case "Value":
if valueAlreadyParsed {
return errDuplicatedXMLTag
}
tag.Value = s
valueAlreadyParsed = true
default:
return errUnknownXMLTag
}
}
}
return nil
}
func (tag Tag) String() string {
return tag.Key + "=" + tag.Value
}
// IsEmpty returns whether this tag is empty or not.
func (tag Tag) IsEmpty() bool {
return tag.Key == ""
}
// Validate checks this tag.
func (tag Tag) Validate() error {
if len(tag.Key) == 0 || utf8.RuneCountInString(tag.Key) > 128 {
return errInvalidTagKey
}
if utf8.RuneCountInString(tag.Value) > 256 {
return errInvalidTagValue
}
return nil
}

View File

@@ -0,0 +1,178 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package lifecycle
import (
"encoding/xml"
"time"
)
var (
errTransitionInvalidDays = Errorf("Days must be 0 or greater when used with Transition")
errTransitionInvalidDate = Errorf("Date must be provided in ISO 8601 format")
errTransitionInvalid = Errorf("Exactly one of Days (0 or greater) or Date (positive ISO 8601 format) should be present in Transition.")
errTransitionDateNotMidnight = Errorf("'Date' must be at midnight GMT")
)
// TransitionDate is a embedded type containing time.Time to unmarshal
// Date in Transition
type TransitionDate struct {
time.Time
}
// UnmarshalXML parses date from Transition and validates date format
func (tDate *TransitionDate) UnmarshalXML(d *xml.Decoder, startElement xml.StartElement) error {
var dateStr string
err := d.DecodeElement(&dateStr, &startElement)
if err != nil {
return err
}
// While AWS documentation mentions that the date specified
// must be present in ISO 8601 format, in reality they allow
// users to provide RFC 3339 compliant dates.
trnDate, err := time.Parse(time.RFC3339, dateStr)
if err != nil {
return errTransitionInvalidDate
}
// Allow only date timestamp specifying midnight GMT
hr, m, sec := trnDate.Clock()
nsec := trnDate.Nanosecond()
loc := trnDate.Location()
if hr != 0 || m != 0 || sec != 0 || nsec != 0 || loc.String() != time.UTC.String() {
return errTransitionDateNotMidnight
}
*tDate = TransitionDate{trnDate}
return nil
}
// MarshalXML encodes expiration date if it is non-zero and encodes
// empty string otherwise
func (tDate TransitionDate) MarshalXML(e *xml.Encoder, startElement xml.StartElement) error {
if tDate.IsZero() {
return nil
}
return e.EncodeElement(tDate.Format(time.RFC3339), startElement)
}
// TransitionDays is a type alias to unmarshal Days in Transition
type TransitionDays int
// UnmarshalXML parses number of days from Transition and validates if
// >= 0
func (tDays *TransitionDays) UnmarshalXML(d *xml.Decoder, startElement xml.StartElement) error {
var days int
err := d.DecodeElement(&days, &startElement)
if err != nil {
return err
}
if days < 0 {
return errTransitionInvalidDays
}
*tDays = TransitionDays(days)
return nil
}
// MarshalXML encodes number of days to expire if it is non-zero and
// encodes empty string otherwise
func (tDays TransitionDays) MarshalXML(e *xml.Encoder, startElement xml.StartElement) error {
return e.EncodeElement(int(tDays), startElement)
}
// Transition - transition actions for a rule in lifecycle configuration.
type Transition struct {
XMLName xml.Name `xml:"Transition"`
Days TransitionDays `xml:"Days,omitempty"`
Date TransitionDate `xml:"Date,omitempty"`
StorageClass string `xml:"StorageClass,omitempty"`
set bool
}
// IsEnabled returns if transition is enabled.
func (t Transition) IsEnabled() bool {
return t.set
}
// MarshalXML encodes transition field into an XML form.
func (t Transition) MarshalXML(enc *xml.Encoder, start xml.StartElement) error {
if !t.set {
return nil
}
type transitionWrapper Transition
return enc.EncodeElement(transitionWrapper(t), start)
}
// UnmarshalXML decodes transition field from the XML form.
func (t *Transition) UnmarshalXML(d *xml.Decoder, startElement xml.StartElement) error {
type transitionWrapper Transition
var trw transitionWrapper
err := d.DecodeElement(&trw, &startElement)
if err != nil {
return err
}
*t = Transition(trw)
t.set = true
return nil
}
// Validate - validates the "Transition" element
func (t Transition) Validate() error {
if !t.set {
return nil
}
if !t.IsDateNull() && t.Days > 0 {
return errTransitionInvalid
}
if t.StorageClass == "" {
return errXMLNotWellFormed
}
return nil
}
// IsDateNull returns true if date field is null
func (t Transition) IsDateNull() bool {
return t.Date.IsZero()
}
// IsNull returns true if both date and days fields are null
func (t Transition) IsNull() bool {
return t.StorageClass == ""
}
// NextDue returns upcoming transition date for obj and true if applicable,
// returns false otherwise.
func (t Transition) NextDue(obj ObjectOpts) (time.Time, bool) {
if !obj.IsLatest || t.IsNull() {
return time.Time{}, false
}
if !t.IsDateNull() {
return t.Date.Time, true
}
// Days == 0 indicates immediate tiering, i.e object is eligible for tiering since its creation.
if t.Days == 0 {
return obj.ModTime, true
}
return ExpectedExpiryTime(obj.ModTime, int(t.Days)), true
}

View File

@@ -0,0 +1,93 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package lifecycle
import (
"encoding/xml"
"testing"
)
func TestTransitionUnmarshalXML(t *testing.T) {
trTests := []struct {
input string
err error
}{
{
input: `<Transition>
<Days>0</Days>
<StorageClass>S3TIER-1</StorageClass>
</Transition>`,
err: nil,
},
{
input: `<Transition>
<Days>1</Days>
<Date>2021-01-01T00:00:00Z</Date>
<StorageClass>S3TIER-1</StorageClass>
</Transition>`,
err: errTransitionInvalid,
},
{
input: `<Transition>
<Days>1</Days>
</Transition>`,
err: errXMLNotWellFormed,
},
}
for i, tc := range trTests {
var tr Transition
err := xml.Unmarshal([]byte(tc.input), &tr)
if err != nil {
t.Fatalf("%d: xml unmarshal failed with %v", i+1, err)
}
if err = tr.Validate(); err != tc.err {
t.Fatalf("%d: Invalid transition %v: err %v", i+1, tr, err)
}
}
ntrTests := []struct {
input string
err error
}{
{
input: `<NoncurrentVersionTransition>
<NoncurrentDays>0</NoncurrentDays>
<StorageClass>S3TIER-1</StorageClass>
</NoncurrentVersionTransition>`,
err: nil,
},
{
input: `<NoncurrentVersionTransition>
<Days>1</Days>
</NoncurrentVersionTransition>`,
err: errXMLNotWellFormed,
},
}
for i, tc := range ntrTests {
var ntr NoncurrentVersionTransition
err := xml.Unmarshal([]byte(tc.input), &ntr)
if err != nil {
t.Fatalf("%d: xml unmarshal failed with %v", i+1, err)
}
if err = ntr.Validate(); err != tc.err {
t.Fatalf("%d: Invalid noncurrent version transition %v: err %v", i+1, ntr, err)
}
}
}