OP#792 initial commit after fork
This commit is contained in:
45
internal/bucket/versioning/error.go
Normal file
45
internal/bucket/versioning/error.go
Normal file
@@ -0,0 +1,45 @@
|
||||
// Copyright (c) 2015-2021 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package versioning
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// Error is the generic type for any error happening during tag
|
||||
// parsing.
|
||||
type Error struct {
|
||||
err error
|
||||
}
|
||||
|
||||
// Errorf - formats according to a format specifier and returns
|
||||
// the string as a value that satisfies error of type tagging.Error
|
||||
func Errorf(format string, a ...any) error {
|
||||
return Error{err: fmt.Errorf(format, a...)}
|
||||
}
|
||||
|
||||
// Unwrap the internal error.
|
||||
func (e Error) Unwrap() error { return e.err }
|
||||
|
||||
// Error 'error' compatible method.
|
||||
func (e Error) Error() string {
|
||||
if e.err == nil {
|
||||
return "versioning: cause <nil>"
|
||||
}
|
||||
return e.err.Error()
|
||||
}
|
||||
167
internal/bucket/versioning/versioning.go
Normal file
167
internal/bucket/versioning/versioning.go
Normal file
@@ -0,0 +1,167 @@
|
||||
// Copyright (c) 2015-2021 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package versioning
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
"github.com/minio/pkg/v3/wildcard"
|
||||
)
|
||||
|
||||
// State - enabled/disabled/suspended states
|
||||
// for multifactor and status of versioning.
|
||||
type State string
|
||||
|
||||
// Various supported states
|
||||
const (
|
||||
Enabled State = "Enabled"
|
||||
// Disabled State = "Disabled" only used by MFA Delete not supported yet.
|
||||
Suspended State = "Suspended"
|
||||
)
|
||||
|
||||
var (
|
||||
errExcludedPrefixNotSupported = Errorf("excluded prefixes extension supported only when versioning is enabled")
|
||||
errTooManyExcludedPrefixes = Errorf("too many excluded prefixes")
|
||||
)
|
||||
|
||||
// ExcludedPrefix - holds individual prefixes excluded from being versioned.
|
||||
type ExcludedPrefix struct {
|
||||
Prefix string
|
||||
}
|
||||
|
||||
// Versioning - Configuration for bucket versioning.
|
||||
type Versioning struct {
|
||||
XMLNS string `xml:"xmlns,attr,omitempty"`
|
||||
XMLName xml.Name `xml:"VersioningConfiguration"`
|
||||
// MFADelete State `xml:"MFADelete,omitempty"` // not supported yet.
|
||||
Status State `xml:"Status,omitempty"`
|
||||
// MinIO extension - allows selective, prefix-level versioning exclusion.
|
||||
// Requires versioning to be enabled
|
||||
ExcludedPrefixes []ExcludedPrefix `xml:",omitempty"`
|
||||
ExcludeFolders bool `xml:",omitempty"`
|
||||
}
|
||||
|
||||
// Validate - validates the versioning configuration
|
||||
func (v Versioning) Validate() error {
|
||||
// Not supported yet
|
||||
// switch v.MFADelete {
|
||||
// case Enabled, Disabled:
|
||||
// default:
|
||||
// return Errorf("unsupported MFADelete state %s", v.MFADelete)
|
||||
// }
|
||||
switch v.Status {
|
||||
case Enabled:
|
||||
const maxExcludedPrefixes = 10
|
||||
if len(v.ExcludedPrefixes) > maxExcludedPrefixes {
|
||||
return errTooManyExcludedPrefixes
|
||||
}
|
||||
|
||||
case Suspended:
|
||||
if len(v.ExcludedPrefixes) > 0 {
|
||||
return errExcludedPrefixNotSupported
|
||||
}
|
||||
default:
|
||||
return Errorf("unsupported Versioning status %s", v.Status)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Enabled - returns true if versioning is enabled
|
||||
func (v Versioning) Enabled() bool {
|
||||
return v.Status == Enabled
|
||||
}
|
||||
|
||||
// Versioned returns if 'prefix' has versioning enabled or suspended.
|
||||
func (v Versioning) Versioned(prefix string) bool {
|
||||
return v.PrefixEnabled(prefix) || v.PrefixSuspended(prefix)
|
||||
}
|
||||
|
||||
// PrefixEnabled - returns true if versioning is enabled at the bucket and given
|
||||
// prefix, false otherwise.
|
||||
func (v Versioning) PrefixEnabled(prefix string) bool {
|
||||
if v.Status != Enabled {
|
||||
return false
|
||||
}
|
||||
|
||||
if prefix == "" {
|
||||
return true
|
||||
}
|
||||
if v.ExcludeFolders && strings.HasSuffix(prefix, "/") {
|
||||
return false
|
||||
}
|
||||
|
||||
for _, sprefix := range v.ExcludedPrefixes {
|
||||
// Note: all excluded prefix patterns end with `/` (See Validate)
|
||||
sprefix.Prefix += "*"
|
||||
|
||||
if matched := wildcard.MatchSimple(sprefix.Prefix, prefix); matched {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// Suspended - returns true if versioning is suspended
|
||||
func (v Versioning) Suspended() bool {
|
||||
return v.Status == Suspended
|
||||
}
|
||||
|
||||
// PrefixSuspended - returns true if versioning is suspended at the bucket level
|
||||
// or suspended on the given prefix.
|
||||
func (v Versioning) PrefixSuspended(prefix string) bool {
|
||||
if v.Status == Suspended {
|
||||
return true
|
||||
}
|
||||
if v.Status == Enabled {
|
||||
if prefix == "" {
|
||||
return false
|
||||
}
|
||||
if v.ExcludeFolders && strings.HasSuffix(prefix, "/") {
|
||||
return true
|
||||
}
|
||||
|
||||
for _, sprefix := range v.ExcludedPrefixes {
|
||||
// Note: all excluded prefix patterns end with `/` (See Validate)
|
||||
sprefix.Prefix += "*"
|
||||
if matched := wildcard.MatchSimple(sprefix.Prefix, prefix); matched {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// PrefixesExcluded returns true if v contains one or more excluded object
|
||||
// prefixes or if ExcludeFolders is true.
|
||||
func (v Versioning) PrefixesExcluded() bool {
|
||||
return len(v.ExcludedPrefixes) > 0 || v.ExcludeFolders
|
||||
}
|
||||
|
||||
// ParseConfig - parses data in given reader to VersioningConfiguration.
|
||||
func ParseConfig(reader io.Reader) (*Versioning, error) {
|
||||
var v Versioning
|
||||
if err := xml.NewDecoder(reader).Decode(&v); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := v.Validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &v, nil
|
||||
}
|
||||
241
internal/bucket/versioning/versioning_test.go
Normal file
241
internal/bucket/versioning/versioning_test.go
Normal file
@@ -0,0 +1,241 @@
|
||||
// Copyright (c) 2015-2022 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package versioning
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestParseConfig(t *testing.T) {
|
||||
testcases := []struct {
|
||||
input string
|
||||
err error
|
||||
excludedPrefixes []string
|
||||
excludeFolders bool
|
||||
}{
|
||||
{
|
||||
input: `<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<Status>Enabled</Status>
|
||||
</VersioningConfiguration>`,
|
||||
err: nil,
|
||||
},
|
||||
{
|
||||
input: `<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<Status>Enabled</Status>
|
||||
<ExcludedPrefixes>
|
||||
<Prefix>path/to/my/workload/_staging/</Prefix>
|
||||
</ExcludedPrefixes>
|
||||
<ExcludedPrefixes>
|
||||
<Prefix>path/to/my/workload/_temporary/</Prefix>
|
||||
</ExcludedPrefixes>
|
||||
</VersioningConfiguration>`,
|
||||
err: nil,
|
||||
excludedPrefixes: []string{"path/to/my/workload/_staging/", "path/to/my/workload/_temporary/"},
|
||||
},
|
||||
{
|
||||
input: `<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<Status>Suspended</Status>
|
||||
<ExcludedPrefixes>
|
||||
<Prefix>path/to/my/workload/_staging</Prefix>
|
||||
</ExcludedPrefixes>
|
||||
</VersioningConfiguration>`,
|
||||
err: errExcludedPrefixNotSupported,
|
||||
},
|
||||
{
|
||||
input: `<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<Status>Enabled</Status>
|
||||
<ExcludedPrefixes>
|
||||
<Prefix>path/to/my/workload/_staging/ab/</Prefix>
|
||||
</ExcludedPrefixes>
|
||||
<ExcludedPrefixes>
|
||||
<Prefix>path/to/my/workload/_staging/cd/</Prefix>
|
||||
</ExcludedPrefixes>
|
||||
<ExcludedPrefixes>
|
||||
<Prefix>path/to/my/workload/_staging/ef/</Prefix>
|
||||
</ExcludedPrefixes>
|
||||
<ExcludedPrefixes>
|
||||
<Prefix>path/to/my/workload/_staging/gh/</Prefix>
|
||||
</ExcludedPrefixes>
|
||||
<ExcludedPrefixes>
|
||||
<Prefix>path/to/my/workload/_staging/ij/</Prefix>
|
||||
</ExcludedPrefixes>
|
||||
<ExcludedPrefixes>
|
||||
<Prefix>path/to/my/workload/_staging/kl/</Prefix>
|
||||
</ExcludedPrefixes>
|
||||
<ExcludedPrefixes>
|
||||
<Prefix>path/to/my/workload/_staging/mn/</Prefix>
|
||||
</ExcludedPrefixes>
|
||||
<ExcludedPrefixes>
|
||||
<Prefix>path/to/my/workload/_staging/op/</Prefix>
|
||||
</ExcludedPrefixes>
|
||||
<ExcludedPrefixes>
|
||||
<Prefix>path/to/my/workload/_staging/qr/</Prefix>
|
||||
</ExcludedPrefixes>
|
||||
<ExcludedPrefixes>
|
||||
<Prefix>path/to/my/workload/_staging/st/</Prefix>
|
||||
</ExcludedPrefixes>
|
||||
<ExcludedPrefixes>
|
||||
<Prefix>path/to/my/workload/_staging/uv/</Prefix>
|
||||
</ExcludedPrefixes>
|
||||
</VersioningConfiguration>`,
|
||||
err: errTooManyExcludedPrefixes,
|
||||
},
|
||||
{
|
||||
input: `<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<Status>Enabled</Status>
|
||||
<ExcludeFolders>true</ExcludeFolders>
|
||||
<ExcludedPrefixes>
|
||||
<Prefix>path/to/my/workload/_staging/</Prefix>
|
||||
</ExcludedPrefixes>
|
||||
<ExcludedPrefixes>
|
||||
<Prefix>path/to/my/workload/_temporary/</Prefix>
|
||||
</ExcludedPrefixes>
|
||||
</VersioningConfiguration>`,
|
||||
err: nil,
|
||||
excludedPrefixes: []string{"path/to/my/workload/_staging/", "path/to/my/workload/_temporary/"},
|
||||
excludeFolders: true,
|
||||
},
|
||||
}
|
||||
|
||||
for i, tc := range testcases {
|
||||
var v *Versioning
|
||||
var err error
|
||||
v, err = ParseConfig(strings.NewReader(tc.input))
|
||||
if tc.err != err {
|
||||
t.Fatalf("Test %d: expected %v but got %v", i+1, tc.err, err)
|
||||
}
|
||||
if err != nil {
|
||||
if tc.err == nil {
|
||||
t.Fatalf("Test %d: failed due to %v", i+1, err)
|
||||
}
|
||||
} else {
|
||||
if err := v.Validate(); tc.err != err {
|
||||
t.Fatalf("Test %d: validation failed due to %v", i+1, err)
|
||||
}
|
||||
if len(tc.excludedPrefixes) > 0 {
|
||||
var mismatch bool
|
||||
if len(v.ExcludedPrefixes) != len(tc.excludedPrefixes) {
|
||||
t.Fatalf("Test %d: Expected length of excluded prefixes %d but got %d", i+1, len(tc.excludedPrefixes), len(v.ExcludedPrefixes))
|
||||
}
|
||||
var i int
|
||||
var eprefix string
|
||||
for i, eprefix = range tc.excludedPrefixes {
|
||||
if eprefix != v.ExcludedPrefixes[i].Prefix {
|
||||
mismatch = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if mismatch {
|
||||
t.Fatalf("Test %d: Expected excluded prefix %s but got %s", i+1, tc.excludedPrefixes[i], v.ExcludedPrefixes[i].Prefix)
|
||||
}
|
||||
}
|
||||
if tc.excludeFolders != v.ExcludeFolders {
|
||||
t.Fatalf("Test %d: Expected ExcludeFoldersr=%v but got %v", i+1, tc.excludeFolders, v.ExcludeFolders)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMarshalXML(t *testing.T) {
|
||||
// Validates if Versioning with no excluded prefixes omits
|
||||
// ExcludedPrefixes tags
|
||||
v := Versioning{
|
||||
Status: Enabled,
|
||||
}
|
||||
buf, err := xml.Marshal(v)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to marshal %v: %v", v, err)
|
||||
}
|
||||
|
||||
str := string(buf)
|
||||
if strings.Contains(str, "ExcludedPrefixes") {
|
||||
t.Fatalf("XML shouldn't contain ExcludedPrefixes tag - %s", str)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVersioningZero(t *testing.T) {
|
||||
var v Versioning
|
||||
if v.Enabled() {
|
||||
t.Fatalf("Expected to be disabled but got enabled")
|
||||
}
|
||||
if v.Suspended() {
|
||||
t.Fatalf("Expected to be disabled but got suspended")
|
||||
}
|
||||
}
|
||||
|
||||
func TestExcludeFolders(t *testing.T) {
|
||||
v := Versioning{
|
||||
Status: Enabled,
|
||||
ExcludeFolders: true,
|
||||
}
|
||||
testPrefixes := []string{"jobs/output/_temporary/", "jobs/output/", "jobs/"}
|
||||
for i, prefix := range testPrefixes {
|
||||
if v.PrefixEnabled(prefix) || !v.PrefixSuspended(prefix) {
|
||||
t.Fatalf("Test %d: Expected versioning to be excluded for %s", i+1, prefix)
|
||||
}
|
||||
}
|
||||
|
||||
// Test applicability for regular objects
|
||||
if prefix := "prefix-1/obj-1"; !v.PrefixEnabled(prefix) || v.PrefixSuspended(prefix) {
|
||||
t.Fatalf("Expected versioning to be enabled for %s", prefix)
|
||||
}
|
||||
|
||||
// Test when ExcludeFolders is disabled
|
||||
v.ExcludeFolders = false
|
||||
for i, prefix := range testPrefixes {
|
||||
if !v.PrefixEnabled(prefix) || v.PrefixSuspended(prefix) {
|
||||
t.Fatalf("Test %d: Expected versioning to be enabled for %s", i+1, prefix)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestExcludedPrefixesMatch(t *testing.T) {
|
||||
v := Versioning{
|
||||
Status: Enabled,
|
||||
ExcludedPrefixes: []ExcludedPrefix{{"*/_temporary/"}},
|
||||
}
|
||||
|
||||
if err := v.Validate(); err != nil {
|
||||
t.Fatalf("Invalid test versioning config %v: %v", v, err)
|
||||
}
|
||||
tests := []struct {
|
||||
prefix string
|
||||
excluded bool
|
||||
}{
|
||||
{
|
||||
prefix: "app1-jobs/output/_temporary/attempt1/data.csv",
|
||||
excluded: true,
|
||||
},
|
||||
{
|
||||
prefix: "app1-jobs/output/final/attempt1/data.csv",
|
||||
excluded: false,
|
||||
},
|
||||
}
|
||||
|
||||
for i, test := range tests {
|
||||
if v.PrefixSuspended(test.prefix) != test.excluded {
|
||||
if test.excluded {
|
||||
t.Fatalf("Test %d: Expected prefix %s to be excluded from versioning", i+1, test.prefix)
|
||||
} else {
|
||||
t.Fatalf("Test %d: Expected prefix %s to have versioning enabled", i+1, test.prefix)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user