confdroid/confdroid_minio
1
0
Fork 0
Code Issues Packages Wiki Activity

OP#792 initial commit after fork

Browse Source
This commit is contained in:
Arne Teuke
2026-09-27 11:03:48 +02:00
parent 1fc33f4f1f
commit 55b0890196
1338 changed files with 428609 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files

344
internal/config/api/api.go Normal file
View File

@@ -0,0 +1,344 @@
// Copyright (c) 2015-2023 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package api
import (
"encoding/json"
"errors"
"fmt"
"math"
"slices"
"strconv"
"strings"
"time"
"github.com/minio/minio/internal/config"
"github.com/minio/pkg/v3/env"
)
// API sub-system constants
const (
apiRequestsMax = "requests_max"
apiClusterDeadline = "cluster_deadline"
apiCorsAllowOrigin = "cors_allow_origin"
apiRemoteTransportDeadline = "remote_transport_deadline"
apiListQuorum = "list_quorum"
apiReplicationPriority = "replication_priority"
apiReplicationMaxWorkers = "replication_max_workers"
apiReplicationMaxLWorkers = "replication_max_lrg_workers"
apiTransitionWorkers = "transition_workers"
apiStaleUploadsCleanupInterval = "stale_uploads_cleanup_interval"
apiStaleUploadsExpiry = "stale_uploads_expiry"
apiDeleteCleanupInterval = "delete_cleanup_interval"
apiDisableODirect = "disable_odirect"
apiODirect = "odirect"
apiGzipObjects = "gzip_objects"
apiRootAccess = "root_access"
apiSyncEvents = "sync_events"
apiObjectMaxVersions = "object_max_versions"
EnvAPIRequestsMax = "MINIO_API_REQUESTS_MAX"
EnvAPIRequestsDeadline = "MINIO_API_REQUESTS_DEADLINE"
EnvAPIClusterDeadline = "MINIO_API_CLUSTER_DEADLINE"
EnvAPICorsAllowOrigin = "MINIO_API_CORS_ALLOW_ORIGIN"
EnvAPIRemoteTransportDeadline = "MINIO_API_REMOTE_TRANSPORT_DEADLINE"
EnvAPITransitionWorkers = "MINIO_API_TRANSITION_WORKERS"
EnvAPIListQuorum = "MINIO_API_LIST_QUORUM"
EnvAPISecureCiphers = "MINIO_API_SECURE_CIPHERS" // default config.EnableOn
EnvAPIReplicationPriority = "MINIO_API_REPLICATION_PRIORITY"
EnvAPIReplicationMaxWorkers = "MINIO_API_REPLICATION_MAX_WORKERS"
EnvAPIReplicationMaxLWorkers = "MINIO_API_REPLICATION_MAX_LRG_WORKERS"
EnvAPIStaleUploadsCleanupInterval = "MINIO_API_STALE_UPLOADS_CLEANUP_INTERVAL"
EnvAPIStaleUploadsExpiry = "MINIO_API_STALE_UPLOADS_EXPIRY"
EnvAPIDeleteCleanupInterval = "MINIO_API_DELETE_CLEANUP_INTERVAL"
EnvDeleteCleanupInterval = "MINIO_DELETE_CLEANUP_INTERVAL"
EnvAPIODirect = "MINIO_API_ODIRECT"
EnvAPIDisableODirect = "MINIO_API_DISABLE_ODIRECT"
EnvAPIGzipObjects = "MINIO_API_GZIP_OBJECTS"
EnvAPIRootAccess = "MINIO_API_ROOT_ACCESS" // default config.EnableOn
EnvAPISyncEvents = "MINIO_API_SYNC_EVENTS" // default "off"
EnvAPIObjectMaxVersions = "MINIO_API_OBJECT_MAX_VERSIONS"
EnvAPIObjectMaxVersionsLegacy = "_MINIO_OBJECT_MAX_VERSIONS"
)
// Deprecated key and ENVs
const (
apiReadyDeadline = "ready_deadline"
apiRequestsDeadline = "requests_deadline"
apiReplicationWorkers = "replication_workers"
apiReplicationFailedWorkers = "replication_failed_workers"
)
// DefaultKVS - default storage class config
var (
DefaultKVS = config.KVS{
config.KV{
Key: apiRequestsMax,
Value: "0",
},
config.KV{
Key: apiClusterDeadline,
Value: "10s",
},
config.KV{
Key: apiCorsAllowOrigin,
Value: "*",
},
config.KV{
Key: apiRemoteTransportDeadline,
Value: "2h",
},
config.KV{
Key: apiListQuorum,
Value: "strict",
},
config.KV{
Key: apiReplicationPriority,
Value: "auto",
},
config.KV{
Key: apiReplicationMaxWorkers,
Value: "500",
},
config.KV{
Key: apiReplicationMaxLWorkers,
Value: "10",
},
config.KV{
Key: apiTransitionWorkers,
Value: "100",
},
config.KV{
Key: apiStaleUploadsCleanupInterval,
Value: "6h",
},
config.KV{
Key: apiStaleUploadsExpiry,
Value: "24h",
},
config.KV{
Key: apiDeleteCleanupInterval,
Value: "5m",
},
config.KV{
Key: apiDisableODirect,
Value: "",
HiddenIfEmpty: true,
},
config.KV{
Key: apiODirect,
Value: config.EnableOn,
},
config.KV{
Key: apiGzipObjects,
Value: config.EnableOff,
},
config.KV{
Key: apiRootAccess,
Value: config.EnableOn,
},
config.KV{
Key: apiSyncEvents,
Value: config.EnableOff,
},
config.KV{
Key: apiObjectMaxVersions,
Value: "9223372036854775807",
},
}
)
// Config storage class configuration
type Config struct {
RequestsMax int `json:"requests_max"`
ClusterDeadline time.Duration `json:"cluster_deadline"`
CorsAllowOrigin []string `json:"cors_allow_origin"`
RemoteTransportDeadline time.Duration `json:"remote_transport_deadline"`
ListQuorum string `json:"list_quorum"`
ReplicationPriority string `json:"replication_priority"`
ReplicationMaxWorkers int `json:"replication_max_workers"`
ReplicationMaxLWorkers int `json:"replication_max_lrg_workers"`
TransitionWorkers int `json:"transition_workers"`
StaleUploadsCleanupInterval time.Duration `json:"stale_uploads_cleanup_interval"`
StaleUploadsExpiry time.Duration `json:"stale_uploads_expiry"`
DeleteCleanupInterval time.Duration `json:"delete_cleanup_interval"`
EnableODirect bool `json:"enable_odirect"`
GzipObjects bool `json:"gzip_objects"`
RootAccess bool `json:"root_access"`
SyncEvents bool `json:"sync_events"`
ObjectMaxVersions int64 `json:"object_max_versions"`
}
// UnmarshalJSON - Validate SS and RRS parity when unmarshalling JSON.
func (sCfg *Config) UnmarshalJSON(data []byte) error {
type Alias Config
aux := &struct {
*Alias
}{
Alias: (*Alias)(sCfg),
}
return json.Unmarshal(data, &aux)
}
// LookupConfig - lookup api config and override with valid environment settings if any.
func LookupConfig(kvs config.KVS) (cfg Config, err error) {
deprecatedKeys := []string{
apiReadyDeadline,
apiRequestsDeadline,
"extend_list_cache_life",
apiReplicationWorkers,
apiReplicationFailedWorkers,
"expiry_workers",
}
disableODirect := env.Get(EnvAPIDisableODirect, kvs.Get(apiDisableODirect)) == config.EnableOn
enableODirect := env.Get(EnvAPIODirect, kvs.Get(apiODirect)) == config.EnableOn
gzipObjects := env.Get(EnvAPIGzipObjects, kvs.Get(apiGzipObjects)) == config.EnableOn
rootAccess := env.Get(EnvAPIRootAccess, kvs.Get(apiRootAccess)) == config.EnableOn
cfg = Config{
EnableODirect: enableODirect || !disableODirect,
GzipObjects: gzipObjects,
RootAccess: rootAccess,
}
var corsAllowOrigin []string
corsList := env.Get(EnvAPICorsAllowOrigin, kvs.Get(apiCorsAllowOrigin))
if corsList == "" {
corsAllowOrigin = []string{"*"} // defaults to '*'
} else {
corsAllowOrigin = strings.Split(corsList, ",")
if slices.Contains(corsAllowOrigin, "") {
return cfg, errors.New("invalid cors value")
}
}
cfg.CorsAllowOrigin = corsAllowOrigin
if err = config.CheckValidKeys(config.APISubSys, kvs, DefaultKVS, deprecatedKeys...); err != nil {
return cfg, err
}
// Check environment variables parameters
requestsMax, err := strconv.Atoi(env.Get(EnvAPIRequestsMax, kvs.GetWithDefault(apiRequestsMax, DefaultKVS)))
if err != nil {
return cfg, err
}
cfg.RequestsMax = requestsMax
if requestsMax < 0 {
return cfg, errors.New("invalid API max requests value")
}
clusterDeadline, err := time.ParseDuration(env.Get(EnvAPIClusterDeadline, kvs.GetWithDefault(apiClusterDeadline, DefaultKVS)))
if err != nil {
return cfg, err
}
cfg.ClusterDeadline = clusterDeadline
remoteTransportDeadline, err := time.ParseDuration(env.Get(EnvAPIRemoteTransportDeadline, kvs.GetWithDefault(apiRemoteTransportDeadline, DefaultKVS)))
if err != nil {
return cfg, err
}
cfg.RemoteTransportDeadline = remoteTransportDeadline
listQuorum := env.Get(EnvAPIListQuorum, kvs.GetWithDefault(apiListQuorum, DefaultKVS))
switch listQuorum {
case "strict", "optimal", "reduced", "disk", "auto":
default:
return cfg, fmt.Errorf("invalid value %v for list_quorum: will default to 'strict'", listQuorum)
}
cfg.ListQuorum = listQuorum
replicationPriority := env.Get(EnvAPIReplicationPriority, kvs.GetWithDefault(apiReplicationPriority, DefaultKVS))
switch replicationPriority {
case "slow", "fast", "auto":
default:
return cfg, fmt.Errorf("invalid value %v for replication_priority", replicationPriority)
}
cfg.ReplicationPriority = replicationPriority
replicationMaxWorkers, err := strconv.Atoi(env.Get(EnvAPIReplicationMaxWorkers, kvs.GetWithDefault(apiReplicationMaxWorkers, DefaultKVS)))
if err != nil {
return cfg, err
}
if replicationMaxWorkers <= 0 || replicationMaxWorkers > 500 {
return cfg, config.ErrInvalidReplicationWorkersValue(nil).Msg("Number of replication workers should be between 1 and 500")
}
cfg.ReplicationMaxWorkers = replicationMaxWorkers
replicationMaxLWorkers, err := strconv.Atoi(env.Get(EnvAPIReplicationMaxLWorkers, kvs.GetWithDefault(apiReplicationMaxLWorkers, DefaultKVS)))
if err != nil {
return cfg, err
}
if replicationMaxLWorkers <= 0 || replicationMaxLWorkers > 10 {
return cfg, config.ErrInvalidReplicationWorkersValue(nil).Msg("Number of replication workers for transfers >=128MiB should be between 1 and 10 per node")
}
cfg.ReplicationMaxLWorkers = replicationMaxLWorkers
transitionWorkers, err := strconv.Atoi(env.Get(EnvAPITransitionWorkers, kvs.GetWithDefault(apiTransitionWorkers, DefaultKVS)))
if err != nil {
return cfg, err
}
cfg.TransitionWorkers = transitionWorkers
v := env.Get(EnvAPIDeleteCleanupInterval, kvs.Get(apiDeleteCleanupInterval))
if v == "" {
v = env.Get(EnvDeleteCleanupInterval, kvs.GetWithDefault(apiDeleteCleanupInterval, DefaultKVS))
}
deleteCleanupInterval, err := time.ParseDuration(v)
if err != nil {
return cfg, err
}
cfg.DeleteCleanupInterval = deleteCleanupInterval
staleUploadsCleanupInterval, err := time.ParseDuration(env.Get(EnvAPIStaleUploadsCleanupInterval, kvs.GetWithDefault(apiStaleUploadsCleanupInterval, DefaultKVS)))
if err != nil {
return cfg, err
}
cfg.StaleUploadsCleanupInterval = staleUploadsCleanupInterval
staleUploadsExpiry, err := time.ParseDuration(env.Get(EnvAPIStaleUploadsExpiry, kvs.GetWithDefault(apiStaleUploadsExpiry, DefaultKVS)))
if err != nil {
return cfg, err
}
cfg.StaleUploadsExpiry = staleUploadsExpiry
cfg.SyncEvents = env.Get(EnvAPISyncEvents, kvs.Get(apiSyncEvents)) == config.EnableOn
maxVerStr := env.Get(EnvAPIObjectMaxVersions, "")
if maxVerStr == "" {
maxVerStr = env.Get(EnvAPIObjectMaxVersionsLegacy, kvs.Get(apiObjectMaxVersions))
}
if maxVerStr != "" {
maxVersions, err := strconv.ParseInt(maxVerStr, 10, 64)
if err != nil {
return cfg, err
}
if maxVersions <= 0 {
return cfg, fmt.Errorf("invalid object max versions value: %v", maxVersions)
}
cfg.ObjectMaxVersions = maxVersions
} else {
cfg.ObjectMaxVersions = math.MaxInt64
}
return cfg, nil
}

120
internal/config/api/help.go Normal file
View File

@@ -0,0 +1,120 @@
// Copyright (c) 2015-2023 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package api
import "github.com/minio/minio/internal/config"
var (
defaultHelpPostfix = func(key string) string {
return config.DefaultHelpPostfix(DefaultKVS, key)
}
// Help holds configuration keys and their default values for api subsystem.
Help = config.HelpKVS{
config.HelpKV{
Key: apiRequestsMax,
Description: `set the maximum number of concurrent requests (default: auto)`,
Optional: true,
Type: "number",
},
config.HelpKV{
Key: apiClusterDeadline,
Description: `set the deadline for cluster readiness check` + defaultHelpPostfix(apiClusterDeadline),
Optional: true,
Type: "duration",
},
config.HelpKV{
Key: apiCorsAllowOrigin,
Description: `set comma separated list of origins allowed for CORS requests` + defaultHelpPostfix(apiCorsAllowOrigin),
Optional: true,
Type: "csv",
},
config.HelpKV{
Key: apiRemoteTransportDeadline,
Description: `set the deadline for API requests on remote transports while proxying between federated instances e.g. "2h"` + defaultHelpPostfix(apiRemoteTransportDeadline),
Optional: true,
Type: "duration",
},
config.HelpKV{
Key: apiListQuorum,
Description: `set the acceptable quorum expected for list operations e.g. "optimal", "reduced", "disk", "strict", "auto"` + defaultHelpPostfix(apiListQuorum),
Optional: true,
Type: "string",
},
config.HelpKV{
Key: apiReplicationPriority,
Description: `set replication priority` + defaultHelpPostfix(apiReplicationPriority),
Optional: true,
Type: "string",
},
config.HelpKV{
Key: apiReplicationMaxWorkers,
Description: `set the maximum number of replication workers` + defaultHelpPostfix(apiReplicationMaxWorkers),
Optional: true,
Type: "number",
},
config.HelpKV{
Key: apiTransitionWorkers,
Description: `set the number of transition workers` + defaultHelpPostfix(apiTransitionWorkers),
Optional: true,
Type: "number",
},
config.HelpKV{
Key: apiStaleUploadsExpiry,
Description: `set to expire stale multipart uploads older than this values` + defaultHelpPostfix(apiStaleUploadsExpiry),
Optional: true,
Type: "duration",
},
config.HelpKV{
Key: apiStaleUploadsCleanupInterval,
Description: `set to change intervals when stale multipart uploads are expired` + defaultHelpPostfix(apiStaleUploadsCleanupInterval),
Optional: true,
Type: "duration",
},
config.HelpKV{
Key: apiDeleteCleanupInterval,
Description: `set to change intervals when deleted objects are permanently deleted from ".trash" folder` + defaultHelpPostfix(apiDeleteCleanupInterval),
Optional: true,
Type: "duration",
},
config.HelpKV{
Key: apiODirect,
Description: "set to enable or disable O_DIRECT for writes under special conditions. NOTE: do not disable O_DIRECT without prior testing" + defaultHelpPostfix(apiODirect),
Optional: true,
Type: "boolean",
},
config.HelpKV{
Key: apiRootAccess,
Description: "turn 'off' root credential access for all API calls including s3, admin operations" + defaultHelpPostfix(apiRootAccess),
Optional: true,
Type: "boolean",
},
config.HelpKV{
Key: apiSyncEvents,
Description: "set to enable synchronous bucket notifications" + defaultHelpPostfix(apiSyncEvents),
Optional: true,
Type: "boolean",
},
config.HelpKV{
Key: apiObjectMaxVersions,
Description: "set max allowed number of versions per object" + defaultHelpPostfix(apiObjectMaxVersions),
Optional: true,
Type: "number",
},
}
)

160
internal/config/batch/batch.go Normal file
View File

@@ -0,0 +1,160 @@
// Copyright (c) 2015-2023 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package batch
import (
"sync"
"time"
"github.com/minio/minio/internal/config"
"github.com/minio/pkg/v3/env"
)
// Batch job environment variables
const (
ReplicationWorkersWait = "replication_workers_wait"
KeyRotationWorkersWait = "keyrotation_workers_wait"
ExpirationWorkersWait = "expiration_workers_wait"
EnvReplicationWorkersWait = "MINIO_BATCH_REPLICATION_WORKERS_WAIT"
EnvKeyRotationWorkersWait = "MINIO_BATCH_KEYROTATION_WORKERS_WAIT"
EnvKeyExpirationWorkersWait = "MINIO_BATCH_EXPIRATION_WORKERS_WAIT"
)
var configMu sync.RWMutex
// Config represents the batch job settings.
type Config struct {
ReplicationWorkersWait time.Duration `json:"replicationWorkersWait"`
KeyRotationWorkersWait time.Duration `json:"keyRotationWorkersWait"`
ExpirationWorkersWait time.Duration `json:"expirationWorkersWait"`
}
// ExpirationWait returns the duration for which a batch expiration worker
// would wait before working on next object.
func (opts Config) ExpirationWait() time.Duration {
configMu.RLock()
defer configMu.RUnlock()
return opts.ExpirationWorkersWait
}
// ReplicationWait returns the duration for which a batch replication worker
// would wait before working on next object.
func (opts Config) ReplicationWait() time.Duration {
configMu.RLock()
defer configMu.RUnlock()
return opts.ReplicationWorkersWait
}
// KeyRotationWait returns the duration for which a batch key-rotation worker
// would wait before working on next object.
func (opts Config) KeyRotationWait() time.Duration {
configMu.RLock()
defer configMu.RUnlock()
return opts.KeyRotationWorkersWait
}
// Clone returns a copy of Config value
func (opts Config) Clone() Config {
configMu.RLock()
defer configMu.RUnlock()
return Config{
ReplicationWorkersWait: opts.ReplicationWorkersWait,
KeyRotationWorkersWait: opts.KeyRotationWorkersWait,
ExpirationWorkersWait: opts.ExpirationWorkersWait,
}
}
// Update updates opts with nopts
func (opts *Config) Update(nopts Config) {
configMu.Lock()
defer configMu.Unlock()
opts.ReplicationWorkersWait = nopts.ReplicationWorkersWait
opts.KeyRotationWorkersWait = nopts.KeyRotationWorkersWait
opts.ExpirationWorkersWait = nopts.ExpirationWorkersWait
}
// DefaultKVS - default KV config for batch job settings
var DefaultKVS = config.KVS{
config.KV{
Key: ReplicationWorkersWait,
Value: "0ms", // No wait by default between each replication attempts.
},
config.KV{
Key: KeyRotationWorkersWait,
Value: "0ms", // No wait by default between each key rotation attempts.
},
config.KV{
Key: ExpirationWorkersWait,
Value: "0ms", // No wait by default between each expiration attempts.
},
}
// LookupConfig - lookup config and override with valid environment settings if any.
func LookupConfig(kvs config.KVS) (cfg Config, err error) {
if err = config.CheckValidKeys(config.BatchSubSys, kvs, DefaultKVS); err != nil {
return cfg, err
}
cfg.ReplicationWorkersWait = 0
cfg.KeyRotationWorkersWait = 0
cfg.ExpirationWorkersWait = 0
rduration, err := time.ParseDuration(env.Get(EnvReplicationWorkersWait, kvs.GetWithDefault(ReplicationWorkersWait, DefaultKVS)))
if err != nil {
return cfg, err
}
if rduration < 0 {
return cfg, config.ErrInvalidBatchReplicationWorkersWait(nil)
}
kduration, err := time.ParseDuration(env.Get(EnvKeyRotationWorkersWait, kvs.GetWithDefault(KeyRotationWorkersWait, DefaultKVS)))
if err != nil {
return cfg, err
}
if kduration < 0 {
return cfg, config.ErrInvalidBatchKeyRotationWorkersWait(nil)
}
eduration, err := time.ParseDuration(env.Get(EnvKeyExpirationWorkersWait, kvs.GetWithDefault(ExpirationWorkersWait, DefaultKVS)))
if err != nil {
return cfg, err
}
if eduration < 0 {
return cfg, config.ErrInvalidBatchExpirationWorkersWait(nil)
}
if rduration > 0 {
cfg.ReplicationWorkersWait = rduration
}
if kduration > 0 {
cfg.KeyRotationWorkersWait = kduration
}
if eduration > 0 {
cfg.ExpirationWorkersWait = eduration
}
return cfg, nil
}

49
internal/config/batch/help.go Normal file
View File

@@ -0,0 +1,49 @@
// Copyright (c) 2015-2023 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package batch
import "github.com/minio/minio/internal/config"
// Help template for batch feature.
var (
defaultHelpPostfix = func(key string) string {
return config.DefaultHelpPostfix(DefaultKVS, key)
}
// Help provides help for config values
Help = config.HelpKVS{
config.HelpKV{
Key: ReplicationWorkersWait,
Description: `maximum sleep duration between objects to slow down batch replication operation` + defaultHelpPostfix(ReplicationWorkersWait),
Optional: true,
Type: "duration",
},
config.HelpKV{
Key: KeyRotationWorkersWait,
Description: `maximum sleep duration between objects to slow down batch keyrotation operation` + defaultHelpPostfix(KeyRotationWorkersWait),
Optional: true,
Type: "duration",
},
config.HelpKV{
Key: ExpirationWorkersWait,
Description: "maximum sleep duration between objects to slow down batch expiration operation" + defaultHelpPostfix(ExpirationWorkersWait),
Optional: true,
Type: "duration",
},
}
)

91
internal/config/bool-flag.go Normal file
View File

@@ -0,0 +1,91 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package config
import (
"encoding/json"
"fmt"
"strconv"
"strings"
)
// BoolFlag - wrapper bool type.
type BoolFlag bool
// String - returns string of BoolFlag.
func (bf BoolFlag) String() string {
if bf {
return "on"
}
return "off"
}
// MarshalJSON - converts BoolFlag into JSON data.
func (bf BoolFlag) MarshalJSON() ([]byte, error) {
return json.Marshal(bf.String())
}
// UnmarshalJSON - parses given data into BoolFlag.
func (bf *BoolFlag) UnmarshalJSON(data []byte) (err error) {
var s string
if err = json.Unmarshal(data, &s); err == nil {
b := BoolFlag(true)
if s == "" {
// Empty string is treated as valid.
*bf = b
} else if b, err = ParseBoolFlag(s); err == nil {
*bf = b
}
}
return err
}
// FormatBool prints stringified version of boolean.
func FormatBool(b bool) string {
if b {
return "on"
}
return "off"
}
// ParseBool returns the boolean value represented by the string.
// It accepts 1, t, T, TRUE, true, True, 0, f, F, FALSE, false, False.
// Any other value returns an error.
func ParseBool(str string) (bool, error) {
switch str {
case "1", "t", "T", "true", "TRUE", "True", "on", "ON", "On":
return true, nil
case "0", "f", "F", "false", "FALSE", "False", "off", "OFF", "Off":
return false, nil
}
if strings.EqualFold(str, "enabled") {
return true, nil
}
if strings.EqualFold(str, "disabled") {
return false, nil
}
return false, fmt.Errorf("ParseBool: parsing '%s': %w", str, strconv.ErrSyntax)
}
// ParseBoolFlag - parses string into BoolFlag.
func ParseBoolFlag(s string) (bf BoolFlag, err error) {
b, err := ParseBool(s)
return BoolFlag(b), err
}

129
internal/config/bool-flag_test.go Normal file
View File

@@ -0,0 +1,129 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package config
import (
"testing"
)
// Test BoolFlag.String()
func TestBoolFlagString(t *testing.T) {
var bf BoolFlag
testCases := []struct {
flag BoolFlag
expectedResult string
}{
{bf, "off"},
{BoolFlag(true), "on"},
{BoolFlag(false), "off"},
}
for _, testCase := range testCases {
str := testCase.flag.String()
if testCase.expectedResult != str {
t.Fatalf("expected: %v, got: %v", testCase.expectedResult, str)
}
}
}
// Test BoolFlag.MarshalJSON()
func TestBoolFlagMarshalJSON(t *testing.T) {
var bf BoolFlag
testCases := []struct {
flag BoolFlag
expectedResult string
}{
{bf, `"off"`},
{BoolFlag(true), `"on"`},
{BoolFlag(false), `"off"`},
}
for _, testCase := range testCases {
data, _ := testCase.flag.MarshalJSON()
if testCase.expectedResult != string(data) {
t.Fatalf("expected: %v, got: %v", testCase.expectedResult, string(data))
}
}
}
// Test BoolFlag.UnmarshalJSON()
func TestBoolFlagUnmarshalJSON(t *testing.T) {
testCases := []struct {
data []byte
expectedResult BoolFlag
expectedErr bool
}{
{[]byte(`{}`), BoolFlag(false), true},
{[]byte(`["on"]`), BoolFlag(false), true},
{[]byte(`"junk"`), BoolFlag(false), true},
{[]byte(`""`), BoolFlag(true), false},
{[]byte(`"on"`), BoolFlag(true), false},
{[]byte(`"off"`), BoolFlag(false), false},
{[]byte(`"true"`), BoolFlag(true), false},
{[]byte(`"false"`), BoolFlag(false), false},
{[]byte(`"ON"`), BoolFlag(true), false},
{[]byte(`"OFF"`), BoolFlag(false), false},
}
for _, testCase := range testCases {
var flag BoolFlag
err := (&flag).UnmarshalJSON(testCase.data)
if !testCase.expectedErr && err != nil {
t.Fatalf("error: expected = <nil>, got = %v", err)
}
if testCase.expectedErr && err == nil {
t.Fatalf("error: expected error, got = <nil>")
}
if err == nil && testCase.expectedResult != flag {
t.Fatalf("result: expected: %v, got: %v", testCase.expectedResult, flag)
}
}
}
// Test ParseBoolFlag()
func TestParseBoolFlag(t *testing.T) {
testCases := []struct {
flagStr string
expectedResult BoolFlag
expectedErr bool
}{
{"", BoolFlag(false), true},
{"junk", BoolFlag(false), true},
{"true", BoolFlag(true), false},
{"false", BoolFlag(false), false},
{"ON", BoolFlag(true), false},
{"OFF", BoolFlag(false), false},
{"on", BoolFlag(true), false},
{"off", BoolFlag(false), false},
}
for _, testCase := range testCases {
bf, err := ParseBoolFlag(testCase.flagStr)
if !testCase.expectedErr && err != nil {
t.Fatalf("error: expected = <nil>, got = %v", err)
}
if testCase.expectedErr && err == nil {
t.Fatalf("error: expected error, got = <nil>")
}
if err == nil && testCase.expectedResult != bf {
t.Fatalf("result: expected: %v, got: %v", testCase.expectedResult, bf)
}
}
}

174
internal/config/browser/browser.go Normal file
View File

@@ -0,0 +1,174 @@
// Copyright (c) 2015-2023 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package browser
import (
"fmt"
"strconv"
"sync"
"github.com/minio/minio/internal/config"
"github.com/minio/pkg/v3/env"
)
// Browser sub-system constants
const (
// browserCSPPolicy setting name for Content-Security-Policy response header value
browserCSPPolicy = "csp_policy"
// browserHSTSSeconds setting name for Strict-Transport-Security response header, amount of seconds for 'max-age'
browserHSTSSeconds = "hsts_seconds"
// browserHSTSIncludeSubdomains setting name for Strict-Transport-Security response header 'includeSubDomains' flag (true or false)
browserHSTSIncludeSubdomains = "hsts_include_subdomains"
// browserHSTSPreload setting name for Strict-Transport-Security response header 'preload' flag (true or false)
browserHSTSPreload = "hsts_preload"
// browserReferrerPolicy setting name for Referrer-Policy response header
browserReferrerPolicy = "referrer_policy"
EnvBrowserCSPPolicy = "MINIO_BROWSER_CONTENT_SECURITY_POLICY"
EnvBrowserHSTSSeconds = "MINIO_BROWSER_HSTS_SECONDS"
EnvBrowserHSTSIncludeSubdomains = "MINIO_BROWSER_HSTS_INCLUDE_SUB_DOMAINS"
EnvBrowserHSTSPreload = "MINIO_BROWSER_HSTS_PRELOAD"
EnvBrowserReferrerPolicy = "MINIO_BROWSER_REFERRER_POLICY"
)
// DefaultKVS - default storage class config
var (
DefaultKVS = config.KVS{
config.KV{
Key: browserCSPPolicy,
Value: "default-src 'self' 'unsafe-eval' 'unsafe-inline'; script-src 'self' https://unpkg.com; connect-src 'self' https://unpkg.com;",
},
config.KV{
Key: browserHSTSSeconds,
Value: "0",
},
config.KV{
Key: browserHSTSIncludeSubdomains,
Value: config.EnableOff,
},
config.KV{
Key: browserHSTSPreload,
Value: config.EnableOff,
},
config.KV{
Key: browserReferrerPolicy,
Value: "strict-origin-when-cross-origin",
},
}
)
// configLock is a global lock for browser config
var configLock sync.RWMutex
// Config storage class configuration
type Config struct {
CSPPolicy string `json:"csp_policy"`
HSTSSeconds int `json:"hsts_seconds"`
HSTSIncludeSubdomains bool `json:"hsts_include_subdomains"`
HSTSPreload bool `json:"hsts_preload"`
ReferrerPolicy string `json:"referrer_policy"`
}
// Update Updates browser with new config
func (browseCfg *Config) Update(newCfg Config) {
configLock.Lock()
defer configLock.Unlock()
browseCfg.CSPPolicy = newCfg.CSPPolicy
browseCfg.HSTSSeconds = newCfg.HSTSSeconds
browseCfg.HSTSIncludeSubdomains = newCfg.HSTSIncludeSubdomains
browseCfg.HSTSPreload = newCfg.HSTSPreload
browseCfg.ReferrerPolicy = newCfg.ReferrerPolicy
}
// LookupConfig - lookup api config and override with valid environment settings if any.
func LookupConfig(kvs config.KVS) (cfg Config, err error) {
cfg = Config{
CSPPolicy: env.Get(EnvBrowserCSPPolicy, kvs.GetWithDefault(browserCSPPolicy, DefaultKVS)),
HSTSSeconds: 0,
HSTSIncludeSubdomains: true,
HSTSPreload: true,
ReferrerPolicy: "strict-origin-when-cross-origin",
}
if err = config.CheckValidKeys(config.BrowserSubSys, kvs, DefaultKVS); err != nil {
return cfg, err
}
hstsIncludeSubdomains := env.Get(EnvBrowserHSTSIncludeSubdomains, kvs.GetWithDefault(browserHSTSIncludeSubdomains, DefaultKVS)) == config.EnableOn
hstsPreload := env.Get(EnvBrowserHSTSPreload, kvs.Get(browserHSTSPreload)) == config.EnableOn
hstsSeconds, err := strconv.Atoi(env.Get(EnvBrowserHSTSSeconds, kvs.GetWithDefault(browserHSTSSeconds, DefaultKVS)))
if err != nil {
return cfg, err
}
cfg.HSTSSeconds = hstsSeconds
cfg.HSTSIncludeSubdomains = hstsIncludeSubdomains
cfg.HSTSPreload = hstsPreload
referrerPolicy := env.Get(EnvBrowserReferrerPolicy, kvs.GetWithDefault(browserReferrerPolicy, DefaultKVS))
switch referrerPolicy {
case "no-referrer", "no-referrer-when-downgrade", "origin", "origin-when-cross-origin", "same-origin", "strict-origin", "strict-origin-when-cross-origin", "unsafe-url":
cfg.ReferrerPolicy = referrerPolicy
default:
return cfg, fmt.Errorf("invalid value %v for %s", referrerPolicy, browserReferrerPolicy)
}
return cfg, nil
}
// GetCSPolicy - Get the Content security Policy
func (browseCfg *Config) GetCSPolicy() string {
configLock.RLock()
defer configLock.RUnlock()
return browseCfg.CSPPolicy
}
// GetHSTSSeconds - Get the Content security Policy
func (browseCfg *Config) GetHSTSSeconds() int {
configLock.RLock()
defer configLock.RUnlock()
return browseCfg.HSTSSeconds
}
// IsHSTSIncludeSubdomains - is HSTS 'includeSubdomains' directive enabled
func (browseCfg *Config) IsHSTSIncludeSubdomains() string {
configLock.RLock()
defer configLock.RUnlock()
if browseCfg.HSTSSeconds > 0 && browseCfg.HSTSIncludeSubdomains {
return config.EnableOn
}
return config.EnableOff
}
// IsHSTSPreload - is HSTS 'preload' directive enabled
func (browseCfg *Config) IsHSTSPreload() string {
configLock.RLock()
defer configLock.RUnlock()
if browseCfg.HSTSSeconds > 0 && browseCfg.HSTSPreload {
return config.EnableOn
}
return config.EnableOff
}
// GetReferPolicy - Get the ReferPolicy
func (browseCfg *Config) GetReferPolicy() string {
configLock.RLock()
defer configLock.RUnlock()
return browseCfg.ReferrerPolicy
}

60
internal/config/browser/help.go Normal file
View File

@@ -0,0 +1,60 @@
// Copyright (c) 2015-2023 MinIO, Inc.
//
// # This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package browser
import "github.com/minio/minio/internal/config"
// Help template for browser feature.
var (
defaultHelpPostfix = func(key string) string {
return config.DefaultHelpPostfix(DefaultKVS, key)
}
Help = config.HelpKVS{
config.HelpKV{
Key: browserCSPPolicy,
Description: `set Content-Security-Policy response header value` + defaultHelpPostfix(browserCSPPolicy),
Optional: true,
Type: "string",
},
config.HelpKV{
Key: browserHSTSSeconds,
Description: `set Strict-Transport-Security 'max-age' amount of seconds value` + defaultHelpPostfix(browserHSTSSeconds),
Optional: true,
Type: "number",
},
config.HelpKV{
Key: browserHSTSIncludeSubdomains,
Description: `turn 'on' to set Strict-Transport-Security 'includeSubDomains' directive` + defaultHelpPostfix(browserHSTSIncludeSubdomains),
Optional: true,
Type: "boolean",
},
config.HelpKV{
Key: browserHSTSPreload,
Description: `turn 'on' to set Strict-Transport-Security 'preload' directive` + defaultHelpPostfix(browserHSTSPreload),
Optional: true,
Type: "boolean",
},
config.HelpKV{
Key: browserReferrerPolicy,
Description: `set Referrer-Policy response header value` + defaultHelpPostfix(browserReferrerPolicy),
Optional: true,
Type: "string",
},
}
)

100
internal/config/callhome/callhome.go Normal file
View File

@@ -0,0 +1,100 @@
// Copyright (c) 2015-2022 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package callhome
import (
"sync"
"time"
"github.com/minio/minio/internal/config"
"github.com/minio/pkg/v3/env"
)
// Callhome related keys
const (
Enable = "enable"
Frequency = "frequency"
)
// DefaultKVS - default KV config for subnet settings
var DefaultKVS = config.KVS{
config.KV{
Key: Enable,
Value: "off",
},
config.KV{
Key: Frequency,
Value: "24h",
},
}
// callhomeCycleDefault is the default interval between two callhome cycles (24hrs)
const callhomeCycleDefault = 24 * time.Hour
// Config represents the subnet related configuration
type Config struct {
// Flag indicating whether callhome is enabled.
Enable bool `json:"enable"`
// The interval between callhome cycles
Frequency time.Duration `json:"frequency"`
}
var configLock sync.RWMutex
// Enabled - indicates if callhome is enabled or not
func (c *Config) Enabled() bool {
configLock.RLock()
defer configLock.RUnlock()
return c.Enable
}
// FrequencyDur - returns the currently configured callhome frequency
func (c *Config) FrequencyDur() time.Duration {
configLock.RLock()
defer configLock.RUnlock()
if c.Frequency == 0 {
return callhomeCycleDefault
}
return c.Frequency
}
// Update updates new callhome frequency
func (c *Config) Update(ncfg Config) {
configLock.Lock()
defer configLock.Unlock()
c.Enable = ncfg.Enable
c.Frequency = ncfg.Frequency
}
// LookupConfig - lookup config and override with valid environment settings if any.
func LookupConfig(kvs config.KVS) (cfg Config, err error) {
if err = config.CheckValidKeys(config.CallhomeSubSys, kvs, DefaultKVS); err != nil {
return cfg, err
}
cfg.Enable = env.Get(config.EnvMinIOCallhomeEnable,
kvs.GetWithDefault(Enable, DefaultKVS)) == config.EnableOn
cfg.Frequency, err = time.ParseDuration(env.Get(config.EnvMinIOCallhomeFrequency,
kvs.GetWithDefault(Frequency, DefaultKVS)))
return cfg, err
}

42
internal/config/callhome/help.go Normal file
View File

@@ -0,0 +1,42 @@
// Copyright (c) 2015-2022 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package callhome
import "github.com/minio/minio/internal/config"
var (
defaultHelpPostfix = func(key string) string {
return config.DefaultHelpPostfix(DefaultKVS, key)
}
// HelpCallhome - provides help for callhome config
HelpCallhome = config.HelpKVS{
config.HelpKV{
Key: Enable,
Type: "on|off",
Description: "set to enable callhome" + defaultHelpPostfix(Enable),
Optional: true,
},
config.HelpKV{
Key: Frequency,
Type: "duration",
Description: "time duration between callhome cycles e.g. 24h" + defaultHelpPostfix(Frequency),
Optional: true,
},
}
)

114
internal/config/certs.go Normal file
View File

@@ -0,0 +1,114 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package config
import (
"bytes"
"crypto/tls"
"crypto/x509"
"encoding/pem"
"errors"
"os"
"github.com/minio/pkg/v3/env"
)
// EnvCertPassword is the environment variable which contains the password used
// to decrypt the TLS private key. It must be set if the TLS private key is
// password protected.
const EnvCertPassword = "MINIO_CERT_PASSWD"
// ParsePublicCertFile - parses public cert into its *x509.Certificate equivalent.
func ParsePublicCertFile(certFile string) (x509Certs []*x509.Certificate, err error) {
// Read certificate file.
var data []byte
if data, err = os.ReadFile(certFile); err != nil {
return nil, err
}
// Trimming leading and tailing white spaces.
data = bytes.TrimSpace(data)
// Parse all certs in the chain.
current := data
for len(current) > 0 {
var pemBlock *pem.Block
if pemBlock, current = pem.Decode(current); pemBlock == nil {
return nil, ErrTLSUnexpectedData(nil).Msgf("Could not read PEM block from file %s", certFile)
}
var x509Cert *x509.Certificate
if x509Cert, err = x509.ParseCertificate(pemBlock.Bytes); err != nil {
return nil, ErrTLSUnexpectedData(nil).Msgf("Failed to parse `%s`: %s", certFile, err.Error())
}
x509Certs = append(x509Certs, x509Cert)
}
if len(x509Certs) == 0 {
return nil, ErrTLSUnexpectedData(nil).Msgf("Empty public certificate file %s", certFile)
}
return x509Certs, nil
}
// LoadX509KeyPair - load an X509 key pair (private key , certificate)
// from the provided paths. The private key may be encrypted and is
// decrypted using the ENV_VAR: MINIO_CERT_PASSWD.
func LoadX509KeyPair(certFile, keyFile string) (tls.Certificate, error) {
certPEMBlock, err := os.ReadFile(certFile)
if err != nil {
return tls.Certificate{}, ErrTLSReadError(nil).Msgf("Unable to read the public key: %s", err)
}
keyPEMBlock, err := os.ReadFile(keyFile)
if err != nil {
return tls.Certificate{}, ErrTLSReadError(nil).Msgf("Unable to read the private key: %s", err)
}
key, rest := pem.Decode(keyPEMBlock)
if len(rest) > 0 {
return tls.Certificate{}, ErrTLSUnexpectedData(nil).Msgf("The private key contains additional data")
}
if key == nil {
return tls.Certificate{}, ErrTLSUnexpectedData(nil).Msgf("The private key is not readable")
}
if x509.IsEncryptedPEMBlock(key) {
password := env.Get(EnvCertPassword, "")
if len(password) == 0 {
return tls.Certificate{}, ErrTLSNoPassword(nil)
}
decryptedKey, decErr := x509.DecryptPEMBlock(key, []byte(password))
if decErr != nil {
return tls.Certificate{}, ErrTLSWrongPassword(decErr)
}
keyPEMBlock = pem.EncodeToMemory(&pem.Block{Type: key.Type, Bytes: decryptedKey})
}
cert, err := tls.X509KeyPair(certPEMBlock, keyPEMBlock)
if err != nil {
return tls.Certificate{}, ErrTLSUnexpectedData(nil).Msg(err.Error())
}
return cert, nil
}
// EnsureCertAndKey checks if both client certificate and key paths are provided
func EnsureCertAndKey(clientCert, clientKey string) error {
if (clientCert != "" && clientKey == "") ||
(clientCert == "" && clientKey != "") {
return errors.New("cert and key must be specified as a pair")
}
return nil
}

454
internal/config/certs_test.go Normal file
View File

@@ -0,0 +1,454 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package config
import (
"os"
"testing"
)
func createTempFile(t testing.TB, prefix, content string) (tempFile string, err error) {
t.Helper()
var tmpfile *os.File
if tmpfile, err = os.CreateTemp(t.TempDir(), prefix); err != nil {
return tempFile, err
}
if _, err = tmpfile.Write([]byte(content)); err != nil {
return tempFile, err
}
if err = tmpfile.Close(); err != nil {
return tempFile, err
}
tempFile = tmpfile.Name()
return tempFile, err
}
func TestParsePublicCertFile(t *testing.T) {
tempFile1, err := createTempFile(t, "public-cert-file", "")
if err != nil {
t.Fatalf("Unable to create temporary file. %v", err)
}
defer os.Remove(tempFile1)
tempFile2, err := createTempFile(t, "public-cert-file", `-----BEGIN CERTIFICATE-----
MIICdTCCAd4CCQCO5G/W1xcE9TANBgkqhkiG9w0BAQUFADB/MQswCQYDVQQGEwJa
WTEOMAwGA1UECBMFTWluaW8xETAPBgNVBAcTCEludGVybmV0MQ4wDAYDVQQKEwVN
aW5pbzEOMAwGA1UECxMFTWluaW8xDjAMBgNVBAMTBU1pbmlvMR0wGwYJKoZIhvcN
AQkBFg50ZXN0c0BtaW5pby5pbzAeFw0xNjEwMTQxMTM0MjJaFw0xNzEwMTQxMTM0
MjJaMH8xCzAJBgNVBAYTAlpZMQ4wDAYDVQQIEwVNaW5pbzERMA8GA1UEBxMISW50
ZXJuZXQxDjAMBgNVBA-some-junk-Q4wDAYDVQQLEwVNaW5pbzEOMAwGA1UEAxMF
TWluaW8xHTAbBgkqhkiG9w0BCQEWDnRlc3RzQG1pbmlvLmlvMIGfMA0GCSqGSIb3
DQEBAQUAA4GNADCBiQKBgQDwNUYB/Sj79WsUE8qnXzzh2glSzWxUE79sCOpQYK83
HWkrl5WxlG8ZxDR1IQV9Ex/lzigJu8G+KXahon6a+3n5GhNrYRe5kIXHQHz0qvv4
aMulqlnYpvSfC83aaO9GVBtwXS/O4Nykd7QBg4nZlazVmsGk7POOjhpjGShRsqpU
JwIDAQABMA0GCSqGSIb3DQEBBQUAA4GBALqjOA6bD8BEl7hkQ8XwX/owSAL0URDe
nUfCOsXgIIAqgw4uTCLOfCJVZNKmRT+KguvPAQ6Z80vau2UxPX5Q2Q+OHXDRrEnK
FjqSBgLP06Qw7a++bshlWGTt5bHWOneW3EQikedckVuIKPkOCib9yGi4VmBBjdFE
M9ofSEt/bdRD
-----END CERTIFICATE-----`)
if err != nil {
t.Fatalf("Unable to create temporary file. %v", err)
}
defer os.Remove(tempFile2)
tempFile3, err := createTempFile(t, "public-cert-file", `-----BEGIN CERTIFICATE-----
MIICdTCCAd4CCQCO5G/W1xcE9TANBgkqhkiG9w0BAQUFADB/MQswCQYDVQQGEwJa
WTEOMAwGA1UECBMFTWluaW8xETAPBgNVBAcTCEludGVybmV0MQ4wDAYDVQQKEwVN
aW5pbzEOMAwGA1UECxMFTWluaW8xDjAMBgNVBAMTBU1pbmlvMR0wGwYJKoZIhvcN
AQkBFg50ZXN0c0BtaW5pby5pbzAeFw0xNjEwMTQxMTM0MjJaFw0xNzEwMTQxMTM0
MjJaMH8xCzAJBgNVBAYTAlpZMQ4wDAYDVQQIEwVNaW5pbzERMA8GA1UEBxMISW50
ZXJuZXQxDjAMBgNVBAabababababaQ4wDAYDVQQLEwVNaW5pbzEOMAwGA1UEAxMF
TWluaW8xHTAbBgkqhkiG9w0BCQEWDnRlc3RzQG1pbmlvLmlvMIGfMA0GCSqGSIb3
DQEBAQUAA4GNADCBiQKBgQDwNUYB/Sj79WsUE8qnXzzh2glSzWxUE79sCOpQYK83
HWkrl5WxlG8ZxDR1IQV9Ex/lzigJu8G+KXahon6a+3n5GhNrYRe5kIXHQHz0qvv4
aMulqlnYpvSfC83aaO9GVBtwXS/O4Nykd7QBg4nZlazVmsGk7POOjhpjGShRsqpU
JwIDAQABMA0GCSqGSIb3DQEBBQUAA4GBALqjOA6bD8BEl7hkQ8XwX/owSAL0URDe
nUfCOsXgIIAqgw4uTCLOfCJVZNKmRT+KguvPAQ6Z80vau2UxPX5Q2Q+OHXDRrEnK
FjqSBgLP06Qw7a++bshlWGTt5bHWOneW3EQikedckVuIKPkOCib9yGi4VmBBjdFE
M9ofSEt/bdRD
-----END CERTIFICATE-----`)
if err != nil {
t.Fatalf("Unable to create temporary file. %v", err)
}
defer os.Remove(tempFile3)
tempFile4, err := createTempFile(t, "public-cert-file", `-----BEGIN CERTIFICATE-----
MIICdTCCAd4CCQCO5G/W1xcE9TANBgkqhkiG9w0BAQUFADB/MQswCQYDVQQGEwJa
WTEOMAwGA1UECBMFTWluaW8xETAPBgNVBAcTCEludGVybmV0MQ4wDAYDVQQKEwVN
aW5pbzEOMAwGA1UECxMFTWluaW8xDjAMBgNVBAMTBU1pbmlvMR0wGwYJKoZIhvcN
AQkBFg50ZXN0c0BtaW5pby5pbzAeFw0xNjEwMTQxMTM0MjJaFw0xNzEwMTQxMTM0
MjJaMH8xCzAJBgNVBAYTAlpZMQ4wDAYDVQQIEwVNaW5pbzERMA8GA1UEBxMISW50
ZXJuZXQxDjAMBgNVBAoTBU1pbmlvMQ4wDAYDVQQLEwVNaW5pbzEOMAwGA1UEAxMF
TWluaW8xHTAbBgkqhkiG9w0BCQEWDnRlc3RzQG1pbmlvLmlvMIGfMA0GCSqGSIb3
DQEBAQUAA4GNADCBiQKBgQDwNUYB/Sj79WsUE8qnXzzh2glSzWxUE79sCOpQYK83
HWkrl5WxlG8ZxDR1IQV9Ex/lzigJu8G+KXahon6a+3n5GhNrYRe5kIXHQHz0qvv4
aMulqlnYpvSfC83aaO9GVBtwXS/O4Nykd7QBg4nZlazVmsGk7POOjhpjGShRsqpU
JwIDAQABMA0GCSqGSIb3DQEBBQUAA4GBALqjOA6bD8BEl7hkQ8XwX/owSAL0URDe
nUfCOsXgIIAqgw4uTCLOfCJVZNKmRT+KguvPAQ6Z80vau2UxPX5Q2Q+OHXDRrEnK
FjqSBgLP06Qw7a++bshlWGTt5bHWOneW3EQikedckVuIKPkOCib9yGi4VmBBjdFE
M9ofSEt/bdRD
-----END CERTIFICATE-----`)
if err != nil {
t.Fatalf("Unable to create temporary file. %v", err)
}
defer os.Remove(tempFile4)
tempFile5, err := createTempFile(t, "public-cert-file", `-----BEGIN CERTIFICATE-----
MIICdTCCAd4CCQCO5G/W1xcE9TANBgkqhkiG9w0BAQUFADB/MQswCQYDVQQGEwJa
WTEOMAwGA1UECBMFTWluaW8xETAPBgNVBAcTCEludGVybmV0MQ4wDAYDVQQKEwVN
aW5pbzEOMAwGA1UECxMFTWluaW8xDjAMBgNVBAMTBU1pbmlvMR0wGwYJKoZIhvcN
AQkBFg50ZXN0c0BtaW5pby5pbzAeFw0xNjEwMTQxMTM0MjJaFw0xNzEwMTQxMTM0
MjJaMH8xCzAJBgNVBAYTAlpZMQ4wDAYDVQQIEwVNaW5pbzERMA8GA1UEBxMISW50
ZXJuZXQxDjAMBgNVBAoTBU1pbmlvMQ4wDAYDVQQLEwVNaW5pbzEOMAwGA1UEAxMF
TWluaW8xHTAbBgkqhkiG9w0BCQEWDnRlc3RzQG1pbmlvLmlvMIGfMA0GCSqGSIb3
DQEBAQUAA4GNADCBiQKBgQDwNUYB/Sj79WsUE8qnXzzh2glSzWxUE79sCOpQYK83
HWkrl5WxlG8ZxDR1IQV9Ex/lzigJu8G+KXahon6a+3n5GhNrYRe5kIXHQHz0qvv4
aMulqlnYpvSfC83aaO9GVBtwXS/O4Nykd7QBg4nZlazVmsGk7POOjhpjGShRsqpU
JwIDAQABMA0GCSqGSIb3DQEBBQUAA4GBALqjOA6bD8BEl7hkQ8XwX/owSAL0URDe
nUfCOsXgIIAqgw4uTCLOfCJVZNKmRT+KguvPAQ6Z80vau2UxPX5Q2Q+OHXDRrEnK
FjqSBgLP06Qw7a++bshlWGTt5bHWOneW3EQikedckVuIKPkOCib9yGi4VmBBjdFE
M9ofSEt/bdRD
-----END CERTIFICATE-----
-----BEGIN CERTIFICATE-----
MIICdTCCAd4CCQCO5G/W1xcE9TANBgkqhkiG9w0BAQUFADB/MQswCQYDVQQGEwJa
WTEOMAwGA1UECBMFTWluaW8xETAPBgNVBAcTCEludGVybmV0MQ4wDAYDVQQKEwVN
aW5pbzEOMAwGA1UECxMFTWluaW8xDjAMBgNVBAMTBU1pbmlvMR0wGwYJKoZIhvcN
AQkBFg50ZXN0c0BtaW5pby5pbzAeFw0xNjEwMTQxMTM0MjJaFw0xNzEwMTQxMTM0
MjJaMH8xCzAJBgNVBAYTAlpZMQ4wDAYDVQQIEwVNaW5pbzERMA8GA1UEBxMISW50
ZXJuZXQxDjAMBgNVBAoTBU1pbmlvMQ4wDAYDVQQLEwVNaW5pbzEOMAwGA1UEAxMF
TWluaW8xHTAbBgkqhkiG9w0BCQEWDnRlc3RzQG1pbmlvLmlvMIGfMA0GCSqGSIb3
DQEBAQUAA4GNADCBiQKBgQDwNUYB/Sj79WsUE8qnXzzh2glSzWxUE79sCOpQYK83
HWkrl5WxlG8ZxDR1IQV9Ex/lzigJu8G+KXahon6a+3n5GhNrYRe5kIXHQHz0qvv4
aMulqlnYpvSfC83aaO9GVBtwXS/O4Nykd7QBg4nZlazVmsGk7POOjhpjGShRsqpU
JwIDAQABMA0GCSqGSIb3DQEBBQUAA4GBALqjOA6bD8BEl7hkQ8XwX/owSAL0URDe
nUfCOsXgIIAqgw4uTCLOfCJVZNKmRT+KguvPAQ6Z80vau2UxPX5Q2Q+OHXDRrEnK
FjqSBgLP06Qw7a++bshlWGTt5bHWOneW3EQikedckVuIKPkOCib9yGi4VmBBjdFE
M9ofSEt/bdRD
-----END CERTIFICATE-----`)
if err != nil {
t.Fatalf("Unable to create temporary file. %v", err)
}
defer os.Remove(tempFile5)
testCases := []struct {
certFile string
expectedResultLen int
expectedErr bool
}{
{"nonexistent-file", 0, true},
{tempFile1, 0, true},
{tempFile2, 0, true},
{tempFile3, 0, true},
{tempFile4, 1, false},
{tempFile5, 2, false},
}
for _, testCase := range testCases {
certs, err := ParsePublicCertFile(testCase.certFile)
if !testCase.expectedErr && err != nil {
t.Fatalf("error: expected = <nil>, got = %v", err)
}
if testCase.expectedErr && err == nil {
t.Fatal("error: expected err, got = <nil>")
}
if len(certs) != testCase.expectedResultLen {
t.Fatalf("certs: expected = %v, got = %v", testCase.expectedResultLen, len(certs))
}
}
}
func TestLoadX509KeyPair(t *testing.T) {
t.Cleanup(func() {
os.Unsetenv(EnvCertPassword)
})
for i, testCase := range loadX509KeyPairTests {
privateKey, err := createTempFile(t, "private.key", testCase.privateKey)
if err != nil {
t.Fatalf("Test %d: failed to create tmp private key file: %v", i, err)
}
certificate, err := createTempFile(t, "public.crt", testCase.certificate)
if err != nil {
os.Remove(privateKey)
t.Fatalf("Test %d: failed to create tmp certificate file: %v", i, err)
}
if testCase.password != "" {
t.Setenv(EnvCertPassword, testCase.password)
}
_, err = LoadX509KeyPair(certificate, privateKey)
if err != nil && !testCase.shouldFail {
t.Errorf("Test %d: test should succeed but it failed: %v", i, err)
}
if err == nil && testCase.shouldFail {
t.Errorf("Test %d: test should fail but it succeed", i)
}
os.Remove(privateKey)
os.Remove(certificate)
}
}
var loadX509KeyPairTests = []struct {
password string
privateKey, certificate string
shouldFail bool
}{
{
password: "",
privateKey: "",
certificate: `-----BEGIN CERTIFICATE-----
MIIDiTCCAnGgAwIBAgIJAK5m5S7EE46kMA0GCSqGSIb3DQEBCwUAMFsxCzAJBgNV
BAYTAlVTMQ4wDAYDVQQIDAVzdGF0ZTERMA8GA1UEBwwIbG9jYXRpb24xFTATBgNV
BAoMDG9yZ2FuaXphdGlvbjESMBAGA1UEAwwJbG9jYWxob3N0MB4XDTE3MTIxODE4
MDUyOFoXDTI3MTIxNjE4MDUyOFowWzELMAkGA1UEBhMCVVMxDjAMBgNVBAgMBXN0
YXRlMREwDwYDVQQHDAhsb2NhdGlvbjEVMBMGA1UECgwMb3JnYW5pemF0aW9uMRIw
EAYDVQQDDAlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB
AQDPJfYY5Dhsntrqwyu7ZgKM/zrlKEjCwGHhWJBdZdeZCHQlY8ISrtDxxp2XMmI6
HsszalEhNF9fk3vSXWclTuomG03fgGzP4R6QpcwGUCxhRF1J+0b64Yi8pw2uEGsR
GuMwLhGorcWalNoihgHc0BQ4vO8aaTNTX7iD06olesP6vGNu/S8h0VomE+0v9qYc
VF66Zaiv/6OmxAtDpElJjVd0mY7G85BlDlFrVwzd7zhRiuJZ4iDg749Xt9GuuKla
Dvr14glHhP4dQgUbhluJmIHMdx2ZPjk+5FxaDK6I9IUpxczFDe4agDE6lKzU1eLd
cCXRWFOf6q9lTB1hUZfmWfTxAgMBAAGjUDBOMB0GA1UdDgQWBBTQh7lDTq+8salD
0HBNILochiiNaDAfBgNVHSMEGDAWgBTQh7lDTq+8salD0HBNILochiiNaDAMBgNV
HRMEBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQAqi9LycxcXKNSDXaPkCKvw7RQy
iMBDGm1kIY++p3tzbUGuaeu85TsswKnqd50AullEU+aQxRRJGfR8eSKzQJMBXLMQ
b4ptYCc5OrZtRHT8NaZ/df2tc6I88kN8dBu6ybcNGsevXA/iNX3kKLW7naxdr5jj
KUudWSuqDCjCmQa5bYb9H6DreLH2lUItSWBa/YmeZ3VSezDCd+XYO53QKwZVj8Jb
bulZmoo7e7HO1qecEzWKL10UYyEbG3UDPtw+NZc142ZYeEhXQ0dsstGAO5hf3hEl
kQyKGUTpDbKLuyYMFsoH73YLjBqNe+UEhPwE+FWpcky1Sp9RTx/oMLpiZaPR
-----END CERTIFICATE-----`,
shouldFail: true,
},
{
password: "foobar",
privateKey: `-----BEGIN RSA PRIVATE KEY-----
Proc-Type: 4,ENCRYPTED
DEK-Info: AES-128-CBC,CC483BF11678C35F9F02A1AD85DAE285
nMDFd+Qxk1f+S7LwMitmMofNXYNbCY4L1QEqPOOx5wnjNF1wSxmEkL7+h8W4Y/vb
AQt/7TCcUSuSqEMl45nUIcCbhBos5wz+ShvFiez3qKwmR5HSURvqyN6PIJeAbU+h
uw/cvAQsCH1Cq+gYkDJqjrizPhGqg7mSkqyeST3PbOl+ZXc0wynIjA34JSwO3c5j
cF7XKHETtNGj1+AiLruX4wYZAJwQnK375fCoNVMO992zC6K83d8kvGMUgmJjkiIj
q3s4ymFGfoo0S/XNDQXgE5A5QjAKRKUyW2i7pHIIhTyOpeJQeFHDi2/zaZRxoCog
lD2/HKLi5xJtRelZaaGyEJ20c05VzaSZ+EtRIN33foNdyQQL6iAUU3hJ6JlcmRIB
bRfX4XPH1w9UfFU5ZKwUciCoDcL65bsyv/y56ItljBp7Ok+UUKl0H4myFNOSfsuU
IIj4neslnAvwQ8SN4XUpug+7pGF+2m/5UDwRzSUN1H2RfgWN95kqR+tYqCq/E+KO
i0svzFrljSHswsFoPBqKngI7hHwc9QTt5q4frXwj9I4F6HHrTKZnC5M4ef26sbJ1
r7JRmkt0h/GfcS355b0uoBTtF1R8tSJo85Zh47wE+ucdjEvy9/pjnzKqIoJo9bNZ
ri+ue7GhH5EUca1Kd10bH8FqTF+8AHh4yW6xMxSkSgFGp7KtraAVpdp+6kosymqh
dz9VMjA8i28btfkS2isRaCpyumaFYJ3DJMFYhmeyt6gqYovmRLX0qrBf8nrkFTAA
ZmykWsc8ErsCudxlDmKVemuyFL7jtm9IRPq+Jh+IrmixLJFx8PKkNAM6g+A8irx8
piw+yhRsVy5Jk2QeIqvbpxN6BfCNcix4sWkusiCJrAqQFuSm26Mhh53Ig1DXG4d3
6QY1T8tW80Q6JHUtDR+iOPqW6EmrNiEopzirvhGv9FicXZ0Lo2yKJueeeihWhFLL
GmlnCjWVMO4hoo8lWCHv95JkPxGMcecCacKKUbHlXzCGyw3+eeTEHMWMEhziLeBy
HZJ1/GReI3Sx7XlUCkG4468Yz3PpmbNIk/U5XKE7TGuxKmfcWQpu022iF/9DrKTz
KVhKimCBXJX345bCFe1rN2z5CV6sv87FkMs5Y+OjPw6qYFZPVKO2TdUUBcpXbQMg
UW+Kuaax9W7214Stlil727MjRCiH1+0yODg4nWj4pTSocA5R3pn5cwqrjMu97OmL
ESx4DHmy4keeSy3+AIAehCZlwgeLb70/xCSRhJMIMS9Q6bz8CPkEWN8bBZt95oeo
37LqZ7lNmq61fs1x1tq0VUnI9HwLFEnsiubp6RG0Yu8l/uImjjjXa/ytW2GXrfUi
zM22dOntu6u23iBxRBJRWdFTVUz7qrdu+PHavr+Y7TbCeiBwiypmz5llf823UIVx
btamI6ziAq2gKZhObIhut7sjaLkAyTLlNVkNN1WNaplAXpW25UFVk93MHbvZ27bx
9iLGs/qB2kDTUjffSQoHTLY1GoLxv83RgVspUGQjslztEEpWfYvGfVLcgYLv933B
aRW9BRoNZ0czKx7Lhuwjreyb5IcWDarhC8q29ZkkWsQQonaPb0kTEFJul80Yqk0k
-----END RSA PRIVATE KEY-----`,
certificate: `-----BEGIN CERTIFICATE-----
MIIDiTCCAnGgAwIBAgIJAK5m5S7EE46kMA0GCSqGSIb3DQEBCwUAMFsxCzAJBgNV
BAYTAlVTMQ4wDAYDVQQIDAVzdGF0ZTERMA8GA1UEBwwIbG9jYXRpb24xFTATBgNV
BAoMDG9yZ2FuaXphdGlvbjESMBAGA1UEAwwJbG9jYWxob3N0MB4XDTE3MTIxODE4
MDUyOFoXDTI3MTIxNjE4MDUyOFowWzELMAkGA1UEBhMCVVMxDjAMBgNVBAgMBXN0
YXRlMREwDwYDVQQHDAhsb2NhdGlvbjEVMBMGA1UECgwMb3JnYW5pemF0aW9uMRIw
EAYDVQQDDAlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB
AQDPJfYY5Dhsntrqwyu7ZgKM/zrlKEjCwGHhWJBdZdeZCHQlY8ISrtDxxp2XMmI6
HsszalEhNF9fk3vSXWclTuomG03fgGzP4R6QpcwGUCxhRF1J+0b64Yi8pw2uEGsR
GuMwLhGorcWalNoihgHc0BQ4vO8aaTNTX7iD06olesP6vGNu/S8h0VomE+0v9qYc
VF66Zaiv/6OmxAtDpElJjVd0mY7G85BlDlFrVwzd7zhRiuJZ4iDg749Xt9GuuKla
Dvr14glHhP4dQgUbhluJmIHMdx2ZPjk+5FxaDK6I9IUpxczFDe4agDE6lKzU1eLd
cCXRWFOf6q9lTB1hUZfmWfTxAgMBAAGjUDBOMB0GA1UdDgQWBBTQh7lDTq+8salD
0HBNILochiiNaDAfBgNVHSMEGDAWgBTQh7lDTq+8salD0HBNILochiiNaDAMBgNV
HRMEBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQAqi9LycxcXKNSDXaPkCKvw7RQy
iMBDGm1kIY++p3tzbUGuaeu85TsswKnqd50AullEU+aQxRRJGfR8eSKzQJMBXLMQ
b4ptYCc5OrZtRHT8NaZ/df2tc6I88kN8dBu6ybcNGsevXA/iNX3kKLW7naxdr5jj
KUudWSuqDCjCmQa5bYb9H6DreLH2lUItSWBa/YmeZ3VSezDCd+XYO53QKwZVj8Jb
bulZmoo7e7HO1qecEzWKL10UYyEbG3UDPtw+NZc142ZYeEhXQ0dsstGAO5hf3hEl
kQyKGUTpDbKLuyYMFsoH73YLjBqNe+UEhPwE+FWpcky1Sp9RTx/oMLpiZaPR
-----END CERTIFICATE-----`,
shouldFail: false,
},
{
password: "password",
privateKey: `-----BEGIN RSA PRIVATE KEY-----
Proc-Type: 4,ENCRYPTED
DEK-Info: AES-128-CBC,CC483BF11678C35F9F02A1AD85DAE285
nMDFd+Qxk1f+S7LwMitmMofNXYNbCY4L1QEqPOOx5wnjNF1wSxmEkL7+h8W4Y/vb
AQt/7TCcUSuSqEMl45nUIcCbhBos5wz+ShvFiez3qKwmR5HSURvqyN6PIJeAbU+h
uw/cvAQsCH1Cq+gYkDJqjrizPhGqg7mSkqyeST3PbOl+ZXc0wynIjA34JSwO3c5j
cF7XKHETtNGj1+AiLruX4wYZAJwQnK375fCoNVMO992zC6K83d8kvGMUgmJjkiIj
q3s4ymFGfoo0S/XNDQXgE5A5QjAKRKUyW2i7pHIIhTyOpeJQeFHDi2/zaZRxoCog
lD2/HKLi5xJtRelZaaGyEJ20c05VzaSZ+EtRIN33foNdyQQL6iAUU3hJ6JlcmRIB
bRfX4XPH1w9UfFU5ZKwUciCoDcL65bsyv/y56ItljBp7Ok+UUKl0H4myFNOSfsuU
IIj4neslnAvwQ8SN4XUpug+7pGF+2m/5UDwRzSUN1H2RfgWN95kqR+tYqCq/E+KO
i0svzFrljSHswsFoPBqKngI7hHwc9QTt5q4frXwj9I4F6HHrTKZnC5M4ef26sbJ1
r7JRmkt0h/GfcS355b0uoBTtF1R8tSJo85Zh47wE+ucdjEvy9/pjnzKqIoJo9bNZ
ri+ue7GhH5EUca1Kd10bH8FqTF+8AHh4yW6xMxSkSgFGp7KtraAVpdp+6kosymqh
dz9VMjA8i28btfkS2isRaCpyumaFYJ3DJMFYhmeyt6gqYovmRLX0qrBf8nrkFTAA
ZmykWsc8ErsCudxlDmKVemuyFL7jtm9IRPq+Jh+IrmixLJFx8PKkNAM6g+A8irx8
piw+yhRsVy5Jk2QeIqvbpxN6BfCNcix4sWkusiCJrAqQFuSm26Mhh53Ig1DXG4d3
6QY1T8tW80Q6JHUtDR+iOPqW6EmrNiEopzirvhGv9FicXZ0Lo2yKJueeeihWhFLL
GmlnCjWVMO4hoo8lWCHv95JkPxGMcecCacKKUbHlXzCGyw3+eeTEHMWMEhziLeBy
HZJ1/GReI3Sx7XlUCkG4468Yz3PpmbNIk/U5XKE7TGuxKmfcWQpu022iF/9DrKTz
KVhKimCBXJX345bCFe1rN2z5CV6sv87FkMs5Y+OjPw6qYFZPVKO2TdUUBcpXbQMg
UW+Kuaax9W7214Stlil727MjRCiH1+0yODg4nWj4pTSocA5R3pn5cwqrjMu97OmL
ESx4DHmy4keeSy3+AIAehCZlwgeLb70/xCSRhJMIMS9Q6bz8CPkEWN8bBZt95oeo
37LqZ7lNmq61fs1x1tq0VUnI9HwLFEnsiubp6RG0Yu8l/uImjjjXa/ytW2GXrfUi
zM22dOntu6u23iBxRBJRWdFTVUz7qrdu+PHavr+Y7TbCeiBwiypmz5llf823UIVx
btamI6ziAq2gKZhObIhut7sjaLkAyTLlNVkNN1WNaplAXpW25UFVk93MHbvZ27bx
9iLGs/qB2kDTUjffSQoHTLY1GoLxv83RgVspUGQjslztEEpWfYvGfVLcgYLv933B
aRW9BRoNZ0czKx7Lhuwjreyb5IcWDarhC8q29ZkkWsQQonaPb0kTEFJul80Yqk0k
-----END RSA PRIVATE KEY-----`,
certificate: `-----BEGIN CERTIFICATE-----
MIIDiTCCAnGgAwIBAgIJAK5m5S7EE46kMA0GCSqGSIb3DQEBCwUAMFsxCzAJBgNV
BAYTAlVTMQ4wDAYDVQQIDAVzdGF0ZTERMA8GA1UEBwwIbG9jYXRpb24xFTATBgNV
BAoMDG9yZ2FuaXphdGlvbjESMBAGA1UEAwwJbG9jYWxob3N0MB4XDTE3MTIxODE4
MDUyOFoXDTI3MTIxNjE4MDUyOFowWzELMAkGA1UEBhMCVVMxDjAMBgNVBAgMBXN0
YXRlMREwDwYDVQQHDAhsb2NhdGlvbjEVMBMGA1UECgwMb3JnYW5pemF0aW9uMRIw
EAYDVQQDDAlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB
AQDPJfYY5Dhsntrqwyu7ZgKM/zrlKEjCwGHhWJBdZdeZCHQlY8ISrtDxxp2XMmI6
HsszalEhNF9fk3vSXWclTuomG03fgGzP4R6QpcwGUCxhRF1J+0b64Yi8pw2uEGsR
GuMwLhGorcWalNoihgHc0BQ4vO8aaTNTX7iD06olesP6vGNu/S8h0VomE+0v9qYc
VF66Zaiv/6OmxAtDpElJjVd0mY7G85BlDlFrVwzd7zhRiuJZ4iDg749Xt9GuuKla
Dvr14glHhP4dQgUbhluJmIHMdx2ZPjk+5FxaDK6I9IUpxczFDe4agDE6lKzU1eLd
cCXRWFOf6q9lTB1hUZfmWfTxAgMBAAGjUDBOMB0GA1UdDgQWBBTQh7lDTq+8salD
0HBNILochiiNaDAfBgNVHSMEGDAWgBTQh7lDTq+8salD0HBNILochiiNaDAMBgNV
HRMEBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQAqi9LycxcXKNSDXaPkCKvw7RQy
iMBDGm1kIY++p3tzbUGuaeu85TsswKnqd50AullEU+aQxRRJGfR8eSKzQJMBXLMQ
b4ptYCc5OrZtRHT8NaZ/df2tc6I88kN8dBu6ybcNGsevXA/iNX3kKLW7naxdr5jj
KUudWSuqDCjCmQa5bYb9H6DreLH2lUItSWBa/YmeZ3VSezDCd+XYO53QKwZVj8Jb
bulZmoo7e7HO1qecEzWKL10UYyEbG3UDPtw+NZc142ZYeEhXQ0dsstGAO5hf3hEl
kQyKGUTpDbKLuyYMFsoH73YLjBqNe+UEhPwE+FWpcky1Sp9RTx/oMLpiZaPR
-----END CERTIFICATE-----`,
shouldFail: true,
},
{
password: "",
privateKey: `-----BEGIN RSA PRIVATE KEY-----
MIIEpAIBAAKCAQEA4K9Qq7vMY2bGkrdFAYpBYNLlCgnnFU+0pi+N+3bjuWmfX/kw
WXBa3SDqKD08PWWzwvBSLPCCUV2IuUd7tBa1pJ2wXkdoDeI5InYHJKrXbSZonni6
Bex7sgnqV/9o8xFkSOleoQWZgyeKGxtt0J/Z+zhpH+zXahwM4wOL3yzLSQt+NCKM
6N96zXYi16DEa89fYwRxPwE1XTRc7Ddggqx+4iRHvYG0fyTNcPB/+UiFw59EE1Sg
QIyTVntVqpsb6s8XdkFxURoLxefhcMVf2kU0T04OWI3gmeavKfTcj8Z2/bjPSsqP
mgkADv9Ru6VnSK/96TW/NwxWJ32PBz6Sbl9LdwIDAQABAoIBABVh+d5uH/RxyoIZ
+PI9kx1A1NVQvfI0RK/wJKYC2YdCuw0qLOTGIY+b20z7DumU7TenIVrvhKdzrFhd
qjMoWh8RdsByMT/pAKD79JATxi64EgrK2IFJ0TfPY8L+JqHDTPT3aK8QVly5/ZW4
1YmePOOAqdiE9Lc/diaApuYVYD9SL/X7fYs1ezOB4oGXoz0rthX77zHMxcEurpK3
VgSnaq7FYTVY7GrFB+ASiAlDIyLwztz08Ijn8aG0QAZ8GFuPGSmPMXWjLwFhRZsa
Gfy5BYiA0bVSnQSPHzAnHu9HyGlsdouVPPvJB3SrvMl+BFhZiUuR8OGSob7z7hfI
hMyHbNECgYEA/gyG7sHAb5mPkhq9JkTv+LrMY5NDZKYcSlbvBlM3kd6Ib3Hxl+6T
FMq2TWIrh2+mT1C14htziHd05dF6St995Tby6CJxTj6a/2Odnfm+JcOou/ula4Sz
92nIGlGPTJXstDbHGnRCpk6AomXK02stydTyrCisOw1H+LyTG6aT0q8CgYEA4mkO
hfLJkgmJzWIhxHR901uWHz/LId0gC6FQCeaqWmRup6Bl97f0U6xokw4tw8DJOncF
yZpYRXUXhdv/FXCjtXvAhKIX5+e+3dlzPHIdekSfcY00ip/ifAS1OyVviJia+cna
eJgq8WLHxJZim9Ah93NlPyiqGPwtasub90qjZbkCgYEA35WK02o1wII3dvCNc7bM
M+3CoAglEdmXoF1uM/TdPUXKcbqoU3ymeXAGjYhOov3CMp/n0z0xqvLnMLPxmx+i
ny6DDYXyjlhO9WFogHYhwP636+mHJl8+PAsfDvqk0VRJZDmpdUDIv7DrSQGpRfRX
8f+2K4oIOlhv9RuRpI4wHwUCgYB8OjaMyn1NEsy4k2qBt4U+jhcdyEv1pbWqi/U1
qYm5FTgd44VvWVDHBGdQoMv9h28iFCJpzrU2Txv8B4y7v9Ujg+ZLIAFL7j0szt5K
wTZpWvO9Q0Qb98Q2VgL2lADRiyIlglrMJnoRfiisNfOfGKE6e+eGsxI5qUxmN5e5
JQvoiQKBgQCqgyuUBIu/Qsb3qUED/o0S5wCel43Yh/Rl+mxDinOUvJfKJSW2SyEk
+jDo0xw3Opg6ZC5Lj2V809LA/XteaIuyhRuqOopjhHIvIvrYGe+2O8q9/Mv40BYW
0BhJ/Gdseps0C6Z5mTT5Fee4YVlGZuyuNKmKTd4JmqInfBV3ncMWQg==
-----END RSA PRIVATE KEY-----`,
certificate: `-----BEGIN CERTIFICATE-----
MIIDiTCCAnGgAwIBAgIJAIb84Z5Mh31iMA0GCSqGSIb3DQEBCwUAMFsxCzAJBgNV
BAYTAlVTMQ4wDAYDVQQIDAVzdGF0ZTERMA8GA1UEBwwIbG9jYXRpb24xFTATBgNV
BAoMDG9yZ2FuaXphdGlvbjESMBAGA1UEAwwJbG9jYWxob3N0MB4XDTE3MTIxODE4
NTcyM1oXDTI3MTIxNjE4NTcyM1owWzELMAkGA1UEBhMCVVMxDjAMBgNVBAgMBXN0
YXRlMREwDwYDVQQHDAhsb2NhdGlvbjEVMBMGA1UECgwMb3JnYW5pemF0aW9uMRIw
EAYDVQQDDAlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB
AQDgr1Cru8xjZsaSt0UBikFg0uUKCecVT7SmL437duO5aZ9f+TBZcFrdIOooPTw9
ZbPC8FIs8IJRXYi5R3u0FrWknbBeR2gN4jkidgckqtdtJmieeLoF7HuyCepX/2jz
EWRI6V6hBZmDJ4obG23Qn9n7OGkf7NdqHAzjA4vfLMtJC340Iozo33rNdiLXoMRr
z19jBHE/ATVdNFzsN2CCrH7iJEe9gbR/JM1w8H/5SIXDn0QTVKBAjJNWe1Wqmxvq
zxd2QXFRGgvF5+FwxV/aRTRPTg5YjeCZ5q8p9NyPxnb9uM9Kyo+aCQAO/1G7pWdI
r/3pNb83DFYnfY8HPpJuX0t3AgMBAAGjUDBOMB0GA1UdDgQWBBQ2/bSCHscnoV+0
d+YJxLu4XLSNIDAfBgNVHSMEGDAWgBQ2/bSCHscnoV+0d+YJxLu4XLSNIDAMBgNV
HRMEBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQC6p4gPwmkoDtRsP1c8IWgXFka+
Q59oe79ZK1RqDE6ZZu0rgw07rPzKr4ofW4hTxnx7PUgKOhWLq9VvwEC/9tDbD0Gw
SKknRZZOiEE3qUZbwNtHMd4UBzpzChTRC6RcwC5zT1/WICMUHxa4b8E2umJuf3Qd
5Y23sXEESx5evr49z6DLcVe2i70o2wJeWs2kaXqhCJt0X7z0rnYqjfFdvxd8dyzt
1DXmE45cLadpWHDg26DMsdchamgnqEo79YUxkH6G/Cb8ZX4igQ/CsxCDOKvccjHO
OncDtuIpK8O7OyfHP3+MBpUFG4P6Ctn7RVcZe9fQweTpfAy18G+loVzuUeOD
-----END CERTIFICATE-----`,
shouldFail: false,
},
{
password: "foobar",
privateKey: `-----BEGIN RSA PRIVATE KEY-----
MIIEpAIBAAKCAQEA4K9Qq7vMY2bGkrdFAYpBYNLlCgnnFU+0pi+N+3bjuWmfX/kw
WXBa3SDqKD08PWWzwvBSLPCCUV2IuUd7tBa1pJ2wXkdoDeI5InYHJKrXbSZonni6
Bex7sgnqV/9o8xFkSOleoQWZgyeKGxtt0J/Z+zhpH+zXahwM4wOL3yzLSQt+NCKM
6N96zXYi16DEa89fYwRxPwE1XTRc7Ddggqx+4iRHvYG0fyTNcPB/+UiFw59EE1Sg
QIyTVntVqpsb6s8XdkFxURoLxefhcMVf2kU0T04OWI3gmeavKfTcj8Z2/bjPSsqP
mgkADv9Ru6VnSK/96TW/NwxWJ32PBz6Sbl9LdwIDAQABAoIBABVh+d5uH/RxyoIZ
+PI9kx1A1NVQvfI0RK/wJKYC2YdCuw0qLOTGIY+b20z7DumU7TenIVrvhKdzrFhd
qjMoWh8RdsByMT/pAKD79JATxi64EgrK2IFJ0TfPY8L+JqHDTPT3aK8QVly5/ZW4
1YmePOOAqdiE9Lc/diaApuYVYD9SL/X7fYs1ezOB4oGXoz0rthX77zHMxcEurpK3
VgSnaq7FYTVY7GrFB+ASiAlDIyLwztz08Ijn8aG0QAZ8GFuPGSmPMXWjLwFhRZsa
Gfy5BYiA0bVSnQSPHzAnHu9HyGlsdouVPPvJB3SrvMl+BFhZiUuR8OGSob7z7hfI
hMyHbNECgYEA/gyG7sHAb5mPkhq9JkTv+LrMY5NDZKYcSlbvBlM3kd6Ib3Hxl+6T
FMq2TWIrh2+mT1C14htziHd05dF6St995Tby6CJxTj6a/2Odnfm+JcOou/ula4Sz
92nIGlGPTJXstDbHGnRCpk6AomXK02stydTyrCisOw1H+LyTG6aT0q8CgYEA4mkO
hfLJkgmJzWIhxHR901uWHz/LId0gC6FQCeaqWmRup6Bl97f0U6xokw4tw8DJOncF
yZpYRXUXhdv/FXCjtXvAhKIX5+e+3dlzPHIdekSfcY00ip/ifAS1OyVviJia+cna
eJgq8WLHxJZim9Ah93NlPyiqGPwtasub90qjZbkCgYEA35WK02o1wII3dvCNc7bM
M+3CoAglEdmXoF1uM/TdPUXKcbqoU3ymeXAGjYhOov3CMp/n0z0xqvLnMLPxmx+i
ny6DDYXyjlhO9WFogHYhwP636+mHJl8+PAsfDvqk0VRJZDmpdUDIv7DrSQGpRfRX
8f+2K4oIOlhv9RuRpI4wHwUCgYB8OjaMyn1NEsy4k2qBt4U+jhcdyEv1pbWqi/U1
qYm5FTgd44VvWVDHBGdQoMv9h28iFCJpzrU2Txv8B4y7v9Ujg+ZLIAFL7j0szt5K
wTZpWvO9Q0Qb98Q2VgL2lADRiyIlglrMJnoRfiisNfOfGKE6e+eGsxI5qUxmN5e5
JQvoiQKBgQCqgyuUBIu/Qsb3qUED/o0S5wCel43Yh/Rl+mxDinOUvJfKJSW2SyEk
+jDo0xw3Opg6ZC5Lj2V809LA/XteaIuyhRuqOopjhHIvIvrYGe+2O8q9/Mv40BYW
0BhJ/Gdseps0C6Z5mTT5Fee4YVlGZuyuNKmKTd4JmqInfBV3ncMWQg==
-----END RSA PRIVATE KEY-----`,
certificate: `-----BEGIN CERTIFICATE-----
MIIDiTCCAnGgAwIBAgIJAIb84Z5Mh31iMA0GCSqGSIb3DQEBCwUAMFsxCzAJBgNV
BAYTAlVTMQ4wDAYDVQQIDAVzdGF0ZTERMA8GA1UEBwwIbG9jYXRpb24xFTATBgNV
BAoMDG9yZ2FuaXphdGlvbjESMBAGA1UEAwwJbG9jYWxob3N0MB4XDTE3MTIxODE4
NTcyM1oXDTI3MTIxNjE4NTcyM1owWzELMAkGA1UEBhMCVVMxDjAMBgNVBAgMBXN0
YXRlMREwDwYDVQQHDAhsb2NhdGlvbjEVMBMGA1UECgwMb3JnYW5pemF0aW9uMRIw
EAYDVQQDDAlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB
AQDgr1Cru8xjZsaSt0UBikFg0uUKCecVT7SmL437duO5aZ9f+TBZcFrdIOooPTw9
ZbPC8FIs8IJRXYi5R3u0FrWknbBeR2gN4jkidgckqtdtJmieeLoF7HuyCepX/2jz
EWRI6V6hBZmDJ4obG23Qn9n7OGkf7NdqHAzjA4vfLMtJC340Iozo33rNdiLXoMRr
z19jBHE/ATVdNFzsN2CCrH7iJEe9gbR/JM1w8H/5SIXDn0QTVKBAjJNWe1Wqmxvq
zxd2QXFRGgvF5+FwxV/aRTRPTg5YjeCZ5q8p9NyPxnb9uM9Kyo+aCQAO/1G7pWdI
r/3pNb83DFYnfY8HPpJuX0t3AgMBAAGjUDBOMB0GA1UdDgQWBBQ2/bSCHscnoV+0
d+YJxLu4XLSNIDAfBgNVHSMEGDAWgBQ2/bSCHscnoV+0d+YJxLu4XLSNIDAMBgNV
HRMEBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQC6p4gPwmkoDtRsP1c8IWgXFka+
Q59oe79ZK1RqDE6ZZu0rgw07rPzKr4ofW4hTxnx7PUgKOhWLq9VvwEC/9tDbD0Gw
SKknRZZOiEE3qUZbwNtHMd4UBzpzChTRC6RcwC5zT1/WICMUHxa4b8E2umJuf3Qd
5Y23sXEESx5evr49z6DLcVe2i70o2wJeWs2kaXqhCJt0X7z0rnYqjfFdvxd8dyzt
1DXmE45cLadpWHDg26DMsdchamgnqEo79YUxkH6G/Cb8ZX4igQ/CsxCDOKvccjHO
OncDtuIpK8O7OyfHP3+MBpUFG4P6Ctn7RVcZe9fQweTpfAy18G+loVzuUeOD
-----END CERTIFICATE-----`,
shouldFail: false,
},
}

94
internal/config/certsinfo.go Normal file
View File

@@ -0,0 +1,94 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package config
import (
"crypto/x509"
"crypto/x509/pkix"
"fmt"
"net/http"
"strings"
color "github.com/minio/minio/internal/color"
)
// Extra ASN1 OIDs that we may need to handle
var (
oidEmailAddress = []int{1, 2, 840, 113549, 1, 9, 1}
)
// printName prints the fields of a distinguished name, which include such
// things as its common name and locality.
func printName(names []pkix.AttributeTypeAndValue, buf *strings.Builder) []string {
values := []string{}
for _, name := range names {
oid := name.Type
//nolint:gocritic
if len(oid) == 4 && oid[0] == 2 && oid[1] == 5 && oid[2] == 4 {
switch oid[3] {
case 3:
values = append(values, fmt.Sprintf("CN=%s", name.Value))
case 6:
values = append(values, fmt.Sprintf("C=%s", name.Value))
case 8:
values = append(values, fmt.Sprintf("ST=%s", name.Value))
case 10:
values = append(values, fmt.Sprintf("O=%s", name.Value))
case 11:
values = append(values, fmt.Sprintf("OU=%s", name.Value))
default:
values = append(values, fmt.Sprintf("UnknownOID=%s", name.Type.String()))
}
} else if oid.Equal(oidEmailAddress) {
values = append(values, fmt.Sprintf("emailAddress=%s", name.Value))
} else {
values = append(values, fmt.Sprintf("UnknownOID=%s", name.Type.String()))
}
}
if len(values) > 0 {
buf.WriteString(values[0])
for i := 1; i < len(values); i++ {
buf.WriteString(", " + values[i])
}
buf.WriteString("\n")
}
return values
}
// CertificateText returns a human-readable string representation
// of the certificate cert. The format is similar to the OpenSSL
// way of printing certificates (not identical).
func CertificateText(cert *x509.Certificate) string {
var buf strings.Builder
buf.WriteString(color.Blue("\nCertificate:\n"))
if cert.SignatureAlgorithm != x509.UnknownSignatureAlgorithm {
buf.WriteString(color.Blue("%4sSignature Algorithm: ", "") + color.Bold(fmt.Sprintf("%s\n", cert.SignatureAlgorithm)))
}
// Issuer information
buf.WriteString(color.Blue("%4sIssuer: ", ""))
printName(cert.Issuer.Names, &buf)
// Validity information
buf.WriteString(color.Blue("%4sValidity\n", ""))
buf.WriteString(color.Bold(fmt.Sprintf("%8sNot Before: %s\n", "", cert.NotBefore.Format(http.TimeFormat))))
buf.WriteString(color.Bold(fmt.Sprintf("%8sNot After : %s\n", "", cert.NotAfter.Format(http.TimeFormat))))
return buf.String()
}

171
internal/config/compress/compress.go Normal file
View File

@@ -0,0 +1,171 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package compress
import (
"fmt"
"strings"
"github.com/minio/minio/internal/config"
"github.com/minio/pkg/v3/env"
)
// Config represents the compression settings.
type Config struct {
Enabled bool `json:"enabled"`
AllowEncrypted bool `json:"allow_encryption"`
Extensions []string `json:"extensions"`
MimeTypes []string `json:"mime-types"`
}
// Compression environment variables
const (
Extensions = "extensions"
AllowEncrypted = "allow_encryption"
MimeTypes = "mime_types"
EnvCompressState = "MINIO_COMPRESSION_ENABLE"
EnvCompressAllowEncryption = "MINIO_COMPRESSION_ALLOW_ENCRYPTION"
EnvCompressExtensions = "MINIO_COMPRESSION_EXTENSIONS"
EnvCompressMimeTypes = "MINIO_COMPRESSION_MIME_TYPES"
// Include-list for compression.
DefaultExtensions = ".txt,.log,.csv,.json,.tar,.xml,.bin"
DefaultMimeTypes = "text/*,application/json,application/xml,binary/octet-stream"
)
// DefaultKVS - default KV config for compression settings
var (
DefaultKVS = config.KVS{
config.KV{
Key: config.Enable,
Value: config.EnableOff,
},
config.KV{
Key: AllowEncrypted,
Value: config.EnableOff,
},
config.KV{
Key: Extensions,
Value: DefaultExtensions,
},
config.KV{
Key: MimeTypes,
Value: DefaultMimeTypes,
},
}
)
// Parses the given compression exclude list `extensions` or `content-types`.
func parseCompressIncludes(include string) ([]string, error) {
includes := strings.Split(include, config.ValueSeparator)
for _, e := range includes {
if len(e) == 0 {
return nil, config.ErrInvalidCompressionIncludesValue(nil).Msg("extension/mime-type cannot be empty")
}
if e == "/" {
return nil, config.ErrInvalidCompressionIncludesValue(nil).Msg("extension/mime-type cannot be '/'")
}
}
return includes, nil
}
// LookupConfig - lookup compression config.
func LookupConfig(kvs config.KVS) (Config, error) {
var err error
cfg := Config{}
if err = config.CheckValidKeys(config.CompressionSubSys, kvs, DefaultKVS); err != nil {
return cfg, err
}
compress := env.Get(EnvCompressState, "")
if compress == "" {
compress = env.Get(EnvCompressEnableLegacy, "")
if compress == "" {
compress = env.Get(EnvCompress, kvs.Get(config.Enable))
}
}
cfg.Enabled, err = config.ParseBool(compress)
if err != nil {
// Parsing failures happen due to empty KVS, ignore it.
if kvs.Empty() {
return cfg, nil
}
return cfg, err
}
if !cfg.Enabled {
return cfg, nil
}
allowEnc := env.Get(EnvCompressAllowEncryption, "")
if allowEnc == "" {
allowEnc = env.Get(EnvCompressAllowEncryptionLegacy, kvs.Get(AllowEncrypted))
}
cfg.AllowEncrypted, err = config.ParseBool(allowEnc)
if err != nil {
return cfg, err
}
compressExtensions := env.Get(EnvCompressExtensions, kvs.Get(Extensions))
compressExtensionsLegacy := env.Get(EnvCompressExtensionsLegacy, "")
compressMimeTypes := env.Get(EnvCompressMimeTypes, kvs.Get(MimeTypes))
compressMimeTypesLegacy1 := env.Get(EnvCompressMimeTypesLegacy1, "")
compressMimeTypesLegacy2 := env.Get(EnvCompressMimeTypesLegacy2, "")
if compressExtensions != "" {
extensions, err := parseCompressIncludes(compressExtensions)
if err != nil {
return cfg, fmt.Errorf("%s: Invalid MINIO_COMPRESSION_EXTENSIONS value (`%s`)", err, extensions)
}
cfg.Extensions = extensions
}
if compressExtensionsLegacy != "" {
extensions, err := parseCompressIncludes(compressExtensions)
if err != nil {
return cfg, fmt.Errorf("%s: Invalid MINIO_COMPRESS_EXTENSIONS value (`%s`)", err, extensions)
}
cfg.Extensions = extensions
}
if compressMimeTypes != "" {
mimeTypes, err := parseCompressIncludes(compressMimeTypes)
if err != nil {
return cfg, fmt.Errorf("%s: Invalid MINIO_COMPRESSION_MIME_TYPES value (`%s`)", err, mimeTypes)
}
cfg.MimeTypes = mimeTypes
}
if compressMimeTypesLegacy1 != "" {
mimeTypes, err := parseCompressIncludes(compressMimeTypesLegacy1)
if err != nil {
return cfg, fmt.Errorf("%s: Invalid MINIO_COMPRESS_MIMETYPES value (`%s`)", err, mimeTypes)
}
cfg.MimeTypes = mimeTypes
}
if compressMimeTypesLegacy2 != "" {
mimeTypes, err := parseCompressIncludes(compressMimeTypesLegacy2)
if err != nil {
return cfg, fmt.Errorf("%s: Invalid MINIO_COMPRESS_MIME_TYPES value (`%s`)", err, mimeTypes)
}
cfg.MimeTypes = mimeTypes
}
return cfg, nil
}

57
internal/config/compress/compress_test.go Normal file
View File

@@ -0,0 +1,57 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package compress
import (
"reflect"
"testing"
)
func TestParseCompressIncludes(t *testing.T) {
testCases := []struct {
str string
expectedPatterns []string
success bool
}{
// invalid input
{",,,", []string{}, false},
{"", []string{}, false},
{",", []string{}, false},
{"/", []string{}, false},
{"text/*,/", []string{}, false},
// valid input
{".txt,.log", []string{".txt", ".log"}, true},
{"text/*,application/json", []string{"text/*", "application/json"}, true},
}
for _, testCase := range testCases {
t.Run(testCase.str, func(t *testing.T) {
gotPatterns, err := parseCompressIncludes(testCase.str)
if !testCase.success && err == nil {
t.Error("expected failure but success instead")
}
if testCase.success && err != nil {
t.Errorf("expected success but failed instead %s", err)
}
if testCase.success && !reflect.DeepEqual(testCase.expectedPatterns, gotPatterns) {
t.Errorf("expected patterns %s but got %s", testCase.expectedPatterns, gotPatterns)
}
})
}
}

61
internal/config/compress/help.go Normal file
View File

@@ -0,0 +1,61 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package compress
import "github.com/minio/minio/internal/config"
// Help template for compress feature.
var (
defaultHelpPostfix = func(key string) string {
return config.DefaultHelpPostfix(DefaultKVS, key)
}
Help = config.HelpKVS{
config.HelpKV{
Key: config.Enable,
Description: "Enable or disable object compression",
Type: "on|off",
Optional: true,
Sensitive: false,
},
config.HelpKV{
Key: Extensions,
Description: `comma separated file extensions` + defaultHelpPostfix(Extensions),
Optional: true,
Type: "csv",
},
config.HelpKV{
Key: MimeTypes,
Description: `comma separated wildcard mime-types` + defaultHelpPostfix(MimeTypes),
Optional: true,
Type: "csv",
},
config.HelpKV{
Key: AllowEncrypted,
Description: `enable 'encryption' along with compression`,
Optional: true,
Type: "on|off",
},
config.HelpKV{
Key: config.Comment,
Description: config.DefaultComment,
Optional: true,
Type: "sentence",
},
}
)

59
internal/config/compress/legacy.go Normal file
View File

@@ -0,0 +1,59 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package compress
import (
"strings"
"github.com/minio/minio/internal/config"
)
// Legacy envs.
const (
EnvCompress = "MINIO_COMPRESS"
EnvCompressMimeTypesLegacy1 = "MINIO_COMPRESS_MIMETYPES"
// These envs were wrong but we supported them for a long time
// so keep them here to support existing deployments.
EnvCompressEnableLegacy = "MINIO_COMPRESS_ENABLE"
EnvCompressAllowEncryptionLegacy = "MINIO_COMPRESS_ALLOW_ENCRYPTION"
EnvCompressExtensionsLegacy = "MINIO_COMPRESS_EXTENSIONS"
EnvCompressMimeTypesLegacy2 = "MINIO_COMPRESS_MIME_TYPES"
)
// SetCompressionConfig - One time migration code needed, for migrating from older config to new for Compression.
func SetCompressionConfig(s config.Config, cfg Config) {
if !cfg.Enabled {
// No need to save disabled settings in new config.
return
}
s[config.CompressionSubSys][config.Default] = config.KVS{
config.KV{
Key: config.Enable,
Value: config.EnableOn,
},
config.KV{
Key: Extensions,
Value: strings.Join(cfg.Extensions, config.ValueSeparator),
},
config.KV{
Key: MimeTypes,
Value: strings.Join(cfg.MimeTypes, config.ValueSeparator),
},
}
}

1477
internal/config/config.go Normal file
View File

File diff suppressed because it is too large Load Diff

130
internal/config/config_test.go Normal file
View File

@@ -0,0 +1,130 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package config
import (
"testing"
)
func TestKVFields(t *testing.T) {
tests := []struct {
input string
keys []string
expectedFields map[string]struct{}
}{
// No keys present
{
input: "",
keys: []string{"comment"},
expectedFields: map[string]struct{}{},
},
// No keys requested for tokenizing
{
input: `comment="Hi this is my comment ="`,
keys: []string{},
expectedFields: map[string]struct{}{},
},
// Single key requested and present
{
input: `comment="Hi this is my comment ="`,
keys: []string{"comment"},
expectedFields: map[string]struct{}{`comment="Hi this is my comment ="`: {}},
},
// Keys and input order of k=v is same.
{
input: `connection_string="host=localhost port=2832" comment="really long comment"`,
keys: []string{"connection_string", "comment"},
expectedFields: map[string]struct{}{
`connection_string="host=localhost port=2832"`: {},
`comment="really long comment"`: {},
},
},
// Keys with spaces in between
{
input: `enable=on format=namespace connection_string=" host=localhost port=5432 dbname = cesnietor sslmode=disable" table=holicrayoli`,
keys: []string{"enable", "connection_string", "comment", "format", "table"},
expectedFields: map[string]struct{}{
`enable=on`: {},
`format=namespace`: {},
`connection_string=" host=localhost port=5432 dbname = cesnietor sslmode=disable"`: {},
`table=holicrayoli`: {},
},
},
// One of the keys is not present and order of input has changed.
{
input: `comment="really long comment" connection_string="host=localhost port=2832"`,
keys: []string{"connection_string", "comment", "format"},
expectedFields: map[string]struct{}{
`connection_string="host=localhost port=2832"`: {},
`comment="really long comment"`: {},
},
},
// Incorrect delimiter, expected fields should be empty.
{
input: `comment:"really long comment" connection_string:"host=localhost port=2832"`,
keys: []string{"connection_string", "comment"},
expectedFields: map[string]struct{}{},
},
// Incorrect type of input v/s required keys.
{
input: `comme="really long comment" connection_str="host=localhost port=2832"`,
keys: []string{"connection_string", "comment"},
expectedFields: map[string]struct{}{},
},
}
for _, test := range tests {
t.Run("", func(t *testing.T) {
gotFields := kvFields(test.input, test.keys)
if len(gotFields) != len(test.expectedFields) {
t.Errorf("Expected keys %d, found %d", len(test.expectedFields), len(gotFields))
}
found := true
for _, field := range gotFields {
_, ok := test.expectedFields[field]
found = found && ok
}
if !found {
t.Errorf("Expected %s, got %s", test.expectedFields, gotFields)
}
})
}
}
func TestValidRegion(t *testing.T) {
tests := []struct {
name string
success bool
}{
{name: "us-east-1", success: true},
{name: "us_east", success: true},
{name: "helloWorld", success: true},
{name: "-fdslka", success: false},
{name: "^00[", success: false},
{name: "my region", success: false},
{name: "%%$#!", success: false},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
ok := validRegionRegex.MatchString(test.name)
if test.success != ok {
t.Errorf("Expected %t, got %t", test.success, ok)
}
})
}
}

95
internal/config/constants.go Normal file
View File

@@ -0,0 +1,95 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package config
// Config value separator
const (
ValueSeparator = ","
)
// Top level common ENVs
const (
EnvAccessKey = "MINIO_ACCESS_KEY"
EnvSecretKey = "MINIO_SECRET_KEY"
EnvRootUser = "MINIO_ROOT_USER"
EnvRootPassword = "MINIO_ROOT_PASSWORD"
// Legacy files
EnvAccessKeyFile = "MINIO_ACCESS_KEY_FILE"
EnvSecretKeyFile = "MINIO_SECRET_KEY_FILE"
// Current files
EnvRootUserFile = "MINIO_ROOT_USER_FILE"
EnvRootPasswordFile = "MINIO_ROOT_PASSWORD_FILE"
// Set all config environment variables from 'config.env'
// if necessary. Overrides all previous settings and also
// overrides all environment values passed from
// 'podman run -e ENV=value'
EnvConfigEnvFile = "MINIO_CONFIG_ENV_FILE"
EnvBrowser = "MINIO_BROWSER"
EnvDomain = "MINIO_DOMAIN"
EnvPublicIPs = "MINIO_PUBLIC_IPS"
EnvFSOSync = "MINIO_FS_OSYNC"
EnvArgs = "MINIO_ARGS"
EnvVolumes = "MINIO_VOLUMES"
EnvDNSWebhook = "MINIO_DNS_WEBHOOK_ENDPOINT"
EnvSiteName = "MINIO_SITE_NAME"
EnvSiteRegion = "MINIO_SITE_REGION"
EnvMinIOSubnetLicense = "MINIO_SUBNET_LICENSE" // Deprecated Dec 2021
EnvMinIOSubnetAPIKey = "MINIO_SUBNET_API_KEY"
EnvMinIOSubnetProxy = "MINIO_SUBNET_PROXY"
EnvMinIOCallhomeEnable = "MINIO_CALLHOME_ENABLE"
EnvMinIOCallhomeFrequency = "MINIO_CALLHOME_FREQUENCY"
EnvMinIOServerURL = "MINIO_SERVER_URL"
EnvBrowserRedirect = "MINIO_BROWSER_REDIRECT" // On by default
EnvBrowserRedirectURL = "MINIO_BROWSER_REDIRECT_URL"
EnvRootDriveThresholdSize = "MINIO_ROOTDRIVE_THRESHOLD_SIZE"
EnvRootDiskThresholdSize = "MINIO_ROOTDISK_THRESHOLD_SIZE" // Deprecated Sep 2023
EnvBrowserLoginAnimation = "MINIO_BROWSER_LOGIN_ANIMATION"
EnvBrowserSessionDuration = "MINIO_BROWSER_SESSION_DURATION" // Deprecated after November 2023
EnvMinioStsDuration = "MINIO_STS_DURATION"
EnvMinIOLogQueryURL = "MINIO_LOG_QUERY_URL"
EnvMinIOLogQueryAuthToken = "MINIO_LOG_QUERY_AUTH_TOKEN"
EnvMinIOPrometheusURL = "MINIO_PROMETHEUS_URL"
EnvMinIOPrometheusJobID = "MINIO_PROMETHEUS_JOB_ID"
EnvMinIOPrometheusExtraLabels = "MINIO_PROMETHEUS_EXTRA_LABELS"
EnvMinIOPrometheusAuthToken = "MINIO_PROMETHEUS_AUTH_TOKEN"
EnvConsoleDebugLogLevel = "MINIO_CONSOLE_DEBUG_LOGLEVEL"
EnvUpdate = "MINIO_UPDATE"
EnvEndpoints = "MINIO_ENDPOINTS" // legacy
EnvWorm = "MINIO_WORM" // legacy
EnvRegion = "MINIO_REGION" // legacy
EnvRegionName = "MINIO_REGION_NAME" // legacy
)
// Expiration Token durations
// These values are used to validate the expiration time range from
// either the exp claim or MINI_STS_DURATION value
const (
MinExpiration = 900
MaxExpiration = 31536000
)

172
internal/config/crypto.go Normal file
View File

@@ -0,0 +1,172 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package config
import (
"bytes"
"context"
"crypto/rand"
"encoding/binary"
"errors"
"fmt"
"io"
jsoniter "github.com/json-iterator/go"
"github.com/minio/minio/internal/kms"
"github.com/secure-io/sio-go"
"github.com/secure-io/sio-go/sioutil"
)
// EncryptBytes encrypts the plaintext with a key managed by KMS.
// The context is bound to the returned ciphertext.
//
// The same context must be provided when decrypting the
// ciphertext.
func EncryptBytes(k *kms.KMS, plaintext []byte, context kms.Context) ([]byte, error) {
ciphertext, err := Encrypt(k, bytes.NewReader(plaintext), context)
if err != nil {
return nil, err
}
return io.ReadAll(ciphertext)
}
// DecryptBytes decrypts the ciphertext using a key managed by the KMS.
// The same context that have been used during encryption must be
// provided.
func DecryptBytes(k *kms.KMS, ciphertext []byte, context kms.Context) ([]byte, error) {
plaintext, err := Decrypt(k, bytes.NewReader(ciphertext), context)
if err != nil {
return nil, err
}
return io.ReadAll(plaintext)
}
// Encrypt encrypts the plaintext with a key managed by KMS.
// The context is bound to the returned ciphertext.
//
// The same context must be provided when decrypting the
// ciphertext.
func Encrypt(k *kms.KMS, plaintext io.Reader, ctx kms.Context) (io.Reader, error) {
algorithm := sio.AES_256_GCM
if !sioutil.NativeAES() {
algorithm = sio.ChaCha20Poly1305
}
key, err := k.GenerateKey(context.Background(), &kms.GenerateKeyRequest{AssociatedData: ctx})
if err != nil {
return nil, err
}
stream, err := algorithm.Stream(key.Plaintext)
if err != nil {
return nil, err
}
nonce := make([]byte, stream.NonceSize())
if _, err := rand.Read(nonce); err != nil {
return nil, err
}
const (
MaxMetadataSize = 1 << 20 // max. size of the metadata
Version = 1
)
var (
header [5]byte
buffer bytes.Buffer
)
json := jsoniter.ConfigCompatibleWithStandardLibrary
metadata, err := json.Marshal(encryptedObject{
KeyID: key.KeyID,
KMSKey: key.Ciphertext,
Algorithm: algorithm,
Nonce: nonce,
})
if err != nil {
return nil, err
}
if len(metadata) > MaxMetadataSize {
return nil, errors.New("config: encryption metadata is too large")
}
header[0] = Version
binary.LittleEndian.PutUint32(header[1:], uint32(len(metadata)))
buffer.Write(header[:])
buffer.Write(metadata)
return io.MultiReader(
&buffer,
stream.EncryptReader(plaintext, nonce, nil),
), nil
}
// Decrypt decrypts the ciphertext using a key managed by the KMS.
// The same context that have been used during encryption must be
// provided.
func Decrypt(k *kms.KMS, ciphertext io.Reader, associatedData kms.Context) (io.Reader, error) {
const (
MaxMetadataSize = 1 << 20 // max. size of the metadata
Version = 1
)
var header [5]byte
if _, err := io.ReadFull(ciphertext, header[:]); err != nil {
return nil, err
}
if header[0] != Version {
return nil, fmt.Errorf("config: unknown ciphertext version %d", header[0])
}
size := binary.LittleEndian.Uint32(header[1:])
if size > MaxMetadataSize {
return nil, errors.New("config: encryption metadata is too large")
}
var (
metadataBuffer = make([]byte, size)
metadata encryptedObject
)
if _, err := io.ReadFull(ciphertext, metadataBuffer); err != nil {
return nil, err
}
json := jsoniter.ConfigCompatibleWithStandardLibrary
if err := json.Unmarshal(metadataBuffer, &metadata); err != nil {
return nil, err
}
key, err := k.Decrypt(context.TODO(), &kms.DecryptRequest{
Name: metadata.KeyID,
Ciphertext: metadata.KMSKey,
AssociatedData: associatedData,
})
if err != nil {
return nil, err
}
stream, err := metadata.Algorithm.Stream(key)
if err != nil {
return nil, err
}
if stream.NonceSize() != len(metadata.Nonce) {
return nil, sio.NotAuthentic
}
return stream.DecryptReader(ciphertext, metadata.Nonce, nil), nil
}
type encryptedObject struct {
KeyID string `json:"keyid"`
KMSKey []byte `json:"kmskey"`
Algorithm sio.Algorithm `json:"algorithm"`
Nonce []byte `json:"nonce"`
}

118
internal/config/crypto_test.go Normal file
View File

@@ -0,0 +1,118 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package config
import (
"bytes"
"encoding/hex"
"io"
"testing"
"github.com/minio/minio/internal/kms"
)
var encryptDecryptTests = []struct {
Data []byte
Context kms.Context
}{
{
Data: nil,
Context: nil,
},
{
Data: []byte{1},
Context: nil,
},
{
Data: []byte{1},
Context: kms.Context{"key": "value"},
},
{
Data: make([]byte, 1<<20),
Context: kms.Context{"key": "value", "a": "b"},
},
}
func TestEncryptDecrypt(t *testing.T) {
key, err := hex.DecodeString("ddedadb867afa3f73bd33c25499a723ed7f9f51172ee7b1b679e08dc795debcc")
if err != nil {
t.Fatalf("Failed to decode master key: %v", err)
}
KMS, err := kms.NewBuiltin("my-key", key)
if err != nil {
t.Fatalf("Failed to create KMS: %v", err)
}
for i, test := range encryptDecryptTests {
ciphertext, err := Encrypt(KMS, bytes.NewReader(test.Data), test.Context)
if err != nil {
t.Fatalf("Test %d: failed to encrypt stream: %v", i, err)
}
data, err := io.ReadAll(ciphertext)
if err != nil {
t.Fatalf("Test %d: failed to encrypt stream: %v", i, err)
}
plaintext, err := Decrypt(KMS, bytes.NewReader(data), test.Context)
if err != nil {
t.Fatalf("Test %d: failed to decrypt stream: %v", i, err)
}
data, err = io.ReadAll(plaintext)
if err != nil {
t.Fatalf("Test %d: failed to decrypt stream: %v", i, err)
}
if !bytes.Equal(data, test.Data) {
t.Fatalf("Test %d: decrypted data does not match original data", i)
}
}
}
func BenchmarkEncrypt(b *testing.B) {
key, err := hex.DecodeString("ddedadb867afa3f73bd33c25499a723ed7f9f51172ee7b1b679e08dc795debcc")
if err != nil {
b.Fatalf("Failed to decode master key: %v", err)
}
KMS, err := kms.NewBuiltin("my-key", key)
if err != nil {
b.Fatalf("Failed to create KMS: %v", err)
}
benchmarkEncrypt := func(size int, b *testing.B) {
var (
data = make([]byte, size)
plaintext = bytes.NewReader(data)
context = kms.Context{"key": "value"}
)
b.SetBytes(int64(size))
for b.Loop() {
ciphertext, err := Encrypt(KMS, plaintext, context)
if err != nil {
b.Fatal(err)
}
if _, err = io.Copy(io.Discard, ciphertext); err != nil {
b.Fatal(err)
}
plaintext.Reset(data)
}
}
b.Run("1KB", func(b *testing.B) { benchmarkEncrypt(1*1024, b) })
b.Run("512KB", func(b *testing.B) { benchmarkEncrypt(512*1024, b) })
b.Run("1MB", func(b *testing.B) { benchmarkEncrypt(1024*1024, b) })
b.Run("10MB", func(b *testing.B) { benchmarkEncrypt(10*1024*1024, b) })
}

57
internal/config/dns/dns_path.go Normal file
View File

@@ -0,0 +1,57 @@
// Copyright (c) 2015-2024 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package dns
import (
"path"
"strings"
"github.com/miekg/dns"
)
// msgPath converts a domainname to an etcd path. If s looks like service.staging.skydns.local.,
// the resulting key will be /skydns/local/skydns/staging/service .
func msgPath(s, prefix string) string {
l := dns.SplitDomainName(s)
for i, j := 0, len(l)-1; i < j; i, j = i+1, j-1 {
l[i], l[j] = l[j], l[i]
}
return path.Join(append([]string{etcdPathSeparator + prefix + etcdPathSeparator}, l...)...)
}
// dnsJoin joins labels to form a fully qualified domain name. If the last label is
// the root label it is ignored. Not other syntax checks are performed.
func dnsJoin(labels ...string) string {
if len(labels) == 0 {
return ""
}
ll := len(labels)
if labels[ll-1] == "." {
return strings.Join(labels[:ll-1], ".") + "."
}
return dns.Fqdn(strings.Join(labels, "."))
}
// msgUnPath converts a etcd path to domainName.
func msgUnPath(s string) string {
ks := strings.Split(strings.Trim(s, etcdPathSeparator), etcdPathSeparator)
for i, j := 0, len(ks)-1; i < j; i, j = i+1, j-1 {
ks[i], ks[j] = ks[j], ks[i]
}
return strings.Join(ks, ".")
}

291
internal/config/dns/etcd_dns.go Normal file
View File

@@ -0,0 +1,291 @@
// Copyright (c) 2015-2024 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package dns
import (
"context"
"encoding/json"
"errors"
"fmt"
"net"
"sort"
"strings"
"time"
"github.com/minio/minio-go/v7/pkg/set"
clientv3 "go.etcd.io/etcd/client/v3"
)
// ErrNoEntriesFound - Indicates no entries were found for the given key (directory)
var ErrNoEntriesFound = errors.New("No entries found for this key")
// ErrDomainMissing - Indicates domain is missing
var ErrDomainMissing = errors.New("domain is missing")
const etcdPathSeparator = "/"
// create a new coredns service record for the bucket.
func newCoreDNSMsg(ip string, port string, ttl uint32, t time.Time) ([]byte, error) {
return json.Marshal(&SrvRecord{
Host: ip,
Port: json.Number(port),
TTL: ttl,
CreationDate: t,
})
}
// Close closes the internal etcd client and cannot be used further
func (c *CoreDNS) Close() error {
c.etcdClient.Close()
return nil
}
// List - Retrieves list of DNS entries for the domain.
func (c *CoreDNS) List() (map[string][]SrvRecord, error) {
srvRecords := map[string][]SrvRecord{}
for _, domainName := range c.domainNames {
key := msgPath(fmt.Sprintf("%s.", domainName), c.prefixPath)
records, err := c.list(key+etcdPathSeparator, true)
if err != nil {
return srvRecords, err
}
for _, record := range records {
if record.Key == "" {
continue
}
srvRecords[record.Key] = append(srvRecords[record.Key], record)
}
}
return srvRecords, nil
}
// Get - Retrieves DNS records for a bucket.
func (c *CoreDNS) Get(bucket string) ([]SrvRecord, error) {
var srvRecords []SrvRecord
for _, domainName := range c.domainNames {
key := msgPath(fmt.Sprintf("%s.%s.", bucket, domainName), c.prefixPath)
records, err := c.list(key, false)
if err != nil {
return nil, err
}
// Make sure we have record.Key is empty
// this can only happen when record.Key
// has bucket entry with exact prefix
// match any record.Key which do not
// match the prefixes we skip them.
for _, record := range records {
if record.Key != "" {
continue
}
srvRecords = append(srvRecords, record)
}
}
if len(srvRecords) == 0 {
return nil, ErrNoEntriesFound
}
return srvRecords, nil
}
// Retrieves list of entries under the key passed.
// Note that this method fetches entries upto only two levels deep.
func (c *CoreDNS) list(key string, domain bool) ([]SrvRecord, error) {
ctx, cancel := context.WithTimeout(context.Background(), defaultContextTimeout)
r, err := c.etcdClient.Get(ctx, key, clientv3.WithPrefix())
defer cancel()
if err != nil {
return nil, err
}
if r.Count == 0 {
key = strings.TrimSuffix(key, etcdPathSeparator)
r, err = c.etcdClient.Get(ctx, key)
if err != nil {
return nil, err
}
// only if we are looking at `domain` as true
// we should return error here.
if domain && r.Count == 0 {
return nil, ErrDomainMissing
}
}
var srvRecords []SrvRecord
for _, n := range r.Kvs {
var srvRecord SrvRecord
if err = json.Unmarshal(n.Value, &srvRecord); err != nil {
return nil, err
}
srvRecord.Key = strings.TrimPrefix(string(n.Key), key)
srvRecord.Key = strings.TrimSuffix(srvRecord.Key, srvRecord.Host)
// Skip non-bucket entry like for a key
// /skydns/net/miniocloud/10.0.0.1 that may exist as
// dns entry for the server (rather than the bucket
// itself).
if srvRecord.Key == "" {
continue
}
srvRecord.Key = msgUnPath(srvRecord.Key)
srvRecords = append(srvRecords, srvRecord)
}
sort.Slice(srvRecords, func(i int, j int) bool {
return srvRecords[i].Key < srvRecords[j].Key
})
return srvRecords, nil
}
// Put - Adds DNS entries into etcd endpoint in CoreDNS etcd message format.
func (c *CoreDNS) Put(bucket string) error {
c.Delete(bucket) // delete any existing entries.
t := time.Now().UTC()
for ip := range c.domainIPs {
bucketMsg, err := newCoreDNSMsg(ip, c.domainPort, defaultTTL, t)
if err != nil {
return err
}
for _, domainName := range c.domainNames {
key := msgPath(fmt.Sprintf("%s.%s", bucket, domainName), c.prefixPath)
key = key + etcdPathSeparator + ip
ctx, cancel := context.WithTimeout(context.Background(), defaultContextTimeout)
_, err = c.etcdClient.Put(ctx, key, string(bucketMsg))
cancel()
if err != nil {
ctx, cancel = context.WithTimeout(context.Background(), defaultContextTimeout)
c.etcdClient.Delete(ctx, key)
cancel()
return err
}
}
}
return nil
}
// Delete - Removes DNS entries added in Put().
func (c *CoreDNS) Delete(bucket string) error {
for _, domainName := range c.domainNames {
key := msgPath(fmt.Sprintf("%s.%s.", bucket, domainName), c.prefixPath)
ctx, cancel := context.WithTimeout(context.Background(), defaultContextTimeout)
_, err := c.etcdClient.Delete(ctx, key+etcdPathSeparator, clientv3.WithPrefix())
cancel()
if err != nil {
return err
}
}
return nil
}
// DeleteRecord - Removes a specific DNS entry
func (c *CoreDNS) DeleteRecord(record SrvRecord) error {
for _, domainName := range c.domainNames {
key := msgPath(fmt.Sprintf("%s.%s.", record.Key, domainName), c.prefixPath)
ctx, cancel := context.WithTimeout(context.Background(), defaultContextTimeout)
_, err := c.etcdClient.Delete(ctx, key+etcdPathSeparator+record.Host)
cancel()
if err != nil {
return err
}
}
return nil
}
// String stringer name for this implementation of dns.Store
func (c *CoreDNS) String() string {
return "etcdDNS"
}
// CoreDNS - represents dns config for coredns server.
type CoreDNS struct {
domainNames []string
domainIPs set.StringSet
domainPort string
prefixPath string
etcdClient *clientv3.Client
}
// EtcdOption - functional options pattern style
type EtcdOption func(*CoreDNS)
// DomainNames set a list of domain names used by this CoreDNS
// client setting, note this will fail if set to empty when
// constructor initializes.
func DomainNames(domainNames []string) EtcdOption {
return func(args *CoreDNS) {
args.domainNames = domainNames
}
}
// DomainIPs set a list of custom domain IPs, note this will
// fail if set to empty when constructor initializes.
func DomainIPs(domainIPs set.StringSet) EtcdOption {
return func(args *CoreDNS) {
args.domainIPs = domainIPs
}
}
// DomainPort - is a string version of server port
func DomainPort(domainPort string) EtcdOption {
return func(args *CoreDNS) {
args.domainPort = domainPort
}
}
// CoreDNSPath - custom prefix on etcd to populate DNS
// service records, optional and can be empty.
// if empty then c.prefixPath is used i.e "/skydns"
func CoreDNSPath(prefix string) EtcdOption {
return func(args *CoreDNS) {
args.prefixPath = prefix
}
}
// NewCoreDNS - initialize a new coreDNS set/unset values.
func NewCoreDNS(cfg clientv3.Config, setters ...EtcdOption) (Store, error) {
etcdClient, err := clientv3.New(cfg)
if err != nil {
return nil, err
}
args := &CoreDNS{
etcdClient: etcdClient,
}
for _, setter := range setters {
setter(args)
}
if len(args.domainNames) == 0 || args.domainIPs.IsEmpty() {
return nil, errors.New("invalid argument")
}
// strip ports off of domainIPs
domainIPsWithoutPorts := args.domainIPs.ApplyFunc(func(ip string) string {
host, _, err := net.SplitHostPort(ip)
if err != nil {
if strings.Contains(err.Error(), "missing port in address") {
host = ip
}
}
return host
})
args.domainIPs = domainIPsWithoutPorts
return args, nil
}

74
internal/config/dns/etcd_dns_test.go Normal file
View File

@@ -0,0 +1,74 @@
// Copyright (c) 2015-2024 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package dns
import "testing"
func TestDNSJoin(t *testing.T) {
tests := []struct {
in []string
out string
}{
{[]string{"bla", "bliep", "example", "org"}, "bla.bliep.example.org."},
{[]string{"example", "."}, "example."},
{[]string{"example", "org."}, "example.org."}, // technically we should not be called like this.
{[]string{"."}, "."},
}
for i, tc := range tests {
if x := dnsJoin(tc.in...); x != tc.out {
t.Errorf("Test %d, expected %s, got %s", i, tc.out, x)
}
}
}
func TestPath(t *testing.T) {
for _, path := range []string{"mydns", "skydns"} {
result := msgPath("service.staging.skydns.local.", path)
if result != etcdPathSeparator+path+"/local/skydns/staging/service" {
t.Errorf("Failure to get domain's path with prefix: %s", result)
}
}
}
func TestUnPath(t *testing.T) {
result1 := msgUnPath("/skydns/local/cluster/staging/service/")
if result1 != "service.staging.cluster.local.skydns" {
t.Errorf("Failure to get domain from etcd key (with a trailing '/'), expect: 'service.staging.cluster.local.', actually get: '%s'", result1)
}
result2 := msgUnPath("/skydns/local/cluster/staging/service")
if result2 != "service.staging.cluster.local.skydns" {
t.Errorf("Failure to get domain from etcd key (without trailing '/'), expect: 'service.staging.cluster.local.' actually get: '%s'", result2)
}
result3 := msgUnPath("/singleleveldomain/")
if result3 != "singleleveldomain" {
t.Errorf("Failure to get domain from etcd key (with leading and trailing '/'), expect: 'singleleveldomain.' actually get: '%s'", result3)
}
result4 := msgUnPath("/singleleveldomain")
if result4 != "singleleveldomain" {
t.Errorf("Failure to get domain from etcd key (without trailing '/'), expect: 'singleleveldomain.' actually get: '%s'", result4)
}
result5 := msgUnPath("singleleveldomain")
if result5 != "singleleveldomain" {
t.Errorf("Failure to get domain from etcd key (without leading and trailing '/'), expect: 'singleleveldomain.' actually get: '%s'", result5)
}
}

240
internal/config/dns/operator_dns.go Normal file
View File

@@ -0,0 +1,240 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package dns
import (
"context"
"crypto/tls"
"crypto/x509"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/url"
"strconv"
"strings"
"time"
"github.com/golang-jwt/jwt/v4"
"github.com/minio/minio/internal/config"
xhttp "github.com/minio/minio/internal/http"
)
var (
defaultOperatorContextTimeout = 10 * time.Second
// ErrNotImplemented - Indicates the functionality which is not implemented
ErrNotImplemented = errors.New("The method is not implemented")
)
func (c *OperatorDNS) addAuthHeader(r *http.Request) error {
if c.username == "" || c.password == "" {
return nil
}
claims := &jwt.StandardClaims{
ExpiresAt: int64(15 * time.Minute),
Issuer: c.username,
Subject: config.EnvDNSWebhook,
}
token := jwt.NewWithClaims(jwt.SigningMethodHS512, claims)
ss, err := token.SignedString([]byte(c.password))
if err != nil {
return err
}
r.Header.Set("Authorization", "Bearer "+ss)
return nil
}
func (c *OperatorDNS) endpoint(bucket string, del bool) (string, error) {
u, err := url.Parse(c.Endpoint)
if err != nil {
return "", err
}
q := u.Query()
q.Add("bucket", bucket)
q.Add("delete", strconv.FormatBool(del))
u.RawQuery = q.Encode()
return u.String(), nil
}
// Put - Adds DNS entries into operator webhook server
func (c *OperatorDNS) Put(bucket string) error {
ctx, cancel := context.WithTimeout(context.Background(), defaultOperatorContextTimeout)
defer cancel()
e, err := c.endpoint(bucket, false)
if err != nil {
return newError(bucket, err)
}
req, err := http.NewRequestWithContext(ctx, http.MethodPost, e, nil)
if err != nil {
return newError(bucket, err)
}
if err = c.addAuthHeader(req); err != nil {
return newError(bucket, err)
}
resp, err := c.httpClient.Do(req)
if err != nil {
if derr := c.Delete(bucket); derr != nil {
return newError(bucket, derr)
}
return err
}
defer xhttp.DrainBody(resp.Body)
if resp.StatusCode != http.StatusOK {
var errorStringBuilder strings.Builder
io.Copy(&errorStringBuilder, io.LimitReader(resp.Body, resp.ContentLength))
errorString := errorStringBuilder.String()
if resp.StatusCode == http.StatusConflict {
return ErrBucketConflict(Error{bucket, errors.New(errorString)})
}
return newError(bucket, fmt.Errorf("service create for bucket %s, failed with status %s, error %s", bucket, resp.Status, errorString))
}
return nil
}
func newError(bucket string, err error) error {
e := Error{bucket, err}
if strings.Contains(err.Error(), "invalid bucket name") {
return ErrInvalidBucketName(e)
}
return e
}
// Delete - Removes DNS entries added in Put().
func (c *OperatorDNS) Delete(bucket string) error {
ctx, cancel := context.WithTimeout(context.Background(), defaultOperatorContextTimeout)
defer cancel()
e, err := c.endpoint(bucket, true)
if err != nil {
return err
}
req, err := http.NewRequestWithContext(ctx, http.MethodPost, e, nil)
if err != nil {
return err
}
if err = c.addAuthHeader(req); err != nil {
return err
}
resp, err := c.httpClient.Do(req)
if err != nil {
return err
}
xhttp.DrainBody(resp.Body)
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("request to delete the service for bucket %s, failed with status %s", bucket, resp.Status)
}
return nil
}
// DeleteRecord - Removes a specific DNS entry
// No Op for Operator because operator deals on with bucket entries
func (c *OperatorDNS) DeleteRecord(record SrvRecord) error {
return ErrNotImplemented
}
// Close closes the internal http client
func (c *OperatorDNS) Close() error {
return nil
}
// List - Retrieves list of DNS entries for the domain.
// This is a No Op for Operator because, there is no intent to enforce global
// namespace at MinIO level with this DNS entry. The global namespace in
// enforced by the Kubernetes Operator
func (c *OperatorDNS) List() (srvRecords map[string][]SrvRecord, err error) {
return nil, ErrNotImplemented
}
// Get - Retrieves DNS records for a bucket.
// This is a No Op for Operator because, there is no intent to enforce global
// namespace at MinIO level with this DNS entry. The global namespace in
// enforced by the Kubernetes Operator
func (c *OperatorDNS) Get(bucket string) (srvRecords []SrvRecord, err error) {
return nil, ErrNotImplemented
}
// String stringer name for this implementation of dns.Store
func (c *OperatorDNS) String() string {
return "webhookDNS"
}
// OperatorDNS - represents dns config for MinIO k8s operator.
type OperatorDNS struct {
httpClient *http.Client
Endpoint string
rootCAs *x509.CertPool
username string
password string
}
// OperatorOption - functional options pattern style for OperatorDNS
type OperatorOption func(*OperatorDNS)
// Authentication - custom username and password for authenticating at the endpoint
func Authentication(username, password string) OperatorOption {
return func(args *OperatorDNS) {
args.username = username
args.password = password
}
}
// RootCAs - add custom trust certs pool
func RootCAs(certPool *x509.CertPool) OperatorOption {
return func(args *OperatorDNS) {
args.rootCAs = certPool
}
}
// NewOperatorDNS - initialize a new K8S Operator DNS set/unset values.
func NewOperatorDNS(endpoint string, setters ...OperatorOption) (Store, error) {
if endpoint == "" {
return nil, errors.New("invalid argument")
}
args := &OperatorDNS{
Endpoint: endpoint,
}
for _, setter := range setters {
setter(args)
}
args.httpClient = &http.Client{
Transport: &http.Transport{
Proxy: http.ProxyFromEnvironment,
DialContext: (&net.Dialer{
Timeout: 3 * time.Second,
KeepAlive: 5 * time.Second,
}).DialContext,
ResponseHeaderTimeout: 3 * time.Second,
TLSHandshakeTimeout: 3 * time.Second,
ExpectContinueTimeout: 3 * time.Second,
TLSClientConfig: &tls.Config{
RootCAs: args.rootCAs,
},
// Go net/http automatically unzip if content-type is
// gzip disable this feature, as we are always interested
// in raw stream.
DisableCompression: true,
},
}
return args, nil
}

53
internal/config/dns/store.go Normal file
View File

@@ -0,0 +1,53 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package dns
// Error - DNS related errors error.
type Error struct {
Bucket string
Err error
}
// ErrInvalidBucketName for buckets with invalid name
type ErrInvalidBucketName Error
func (e ErrInvalidBucketName) Error() string {
return e.Bucket + " invalid bucket name error: " + e.Err.Error()
}
func (e Error) Error() string {
return "dns related error: " + e.Err.Error()
}
// ErrBucketConflict for buckets that already exist
type ErrBucketConflict Error
func (e ErrBucketConflict) Error() string {
return e.Bucket + " bucket conflict error: " + e.Err.Error()
}
// Store dns record store
type Store interface {
Put(bucket string) error
Get(bucket string) ([]SrvRecord, error)
Delete(bucket string) error
List() (map[string][]SrvRecord, error)
DeleteRecord(record SrvRecord) error
Close() error
String() string
}

57
internal/config/dns/types.go Normal file
View File

@@ -0,0 +1,57 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package dns
import (
"encoding/json"
"time"
)
const (
defaultTTL = 30
defaultContextTimeout = 5 * time.Minute
)
// SrvRecord - represents a DNS service record
type SrvRecord struct {
Host string `json:"host,omitempty"`
Port json.Number `json:"port,omitempty"`
Priority int `json:"priority,omitempty"`
Weight int `json:"weight,omitempty"`
Text string `json:"text,omitempty"`
Mail bool `json:"mail,omitempty"` // Be an MX record. Priority becomes Preference.
TTL uint32 `json:"ttl,omitempty"`
// Holds info about when the entry was created first.
CreationDate time.Time `json:"creationDate"`
// When a SRV record with a "Host: IP-address" is added, we synthesize
// a srv.Target domain name. Normally we convert the full Key where
// the record lives to a DNS name and use this as the srv.Target. When
// TargetStrip > 0 we strip the left most TargetStrip labels from the
// DNS name.
TargetStrip int `json:"targetstrip,omitempty"`
// Group is used to group (or *not* to group) different services
// together. Services with an identical Group are returned in
// the same answer.
Group string `json:"group,omitempty"`
// Key carries the original key used during Put().
Key string `json:"-"`
}

110
internal/config/drive/drive.go Normal file
View File

@@ -0,0 +1,110 @@
// Copyright (c) 2015-2023 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package drive
import (
"sync"
"time"
"github.com/minio/minio/internal/config"
"github.com/minio/pkg/v3/env"
)
// Drive specific timeout environment variables
const (
EnvMaxDriveTimeout = "MINIO_DRIVE_MAX_TIMEOUT"
EnvMaxDriveTimeoutLegacy = "_MINIO_DRIVE_MAX_TIMEOUT"
EnvMaxDiskTimeoutLegacy = "_MINIO_DISK_MAX_TIMEOUT"
)
// DefaultKVS - default KVS for drive
var DefaultKVS = config.KVS{
config.KV{
Key: MaxTimeout,
Value: "30s",
},
}
var configLk sync.RWMutex
// Config represents the subnet related configuration
type Config struct {
// MaxTimeout - maximum timeout for a drive operation
MaxTimeout time.Duration `json:"maxTimeout"`
}
// Update - updates the config with latest values
func (c *Config) Update(updated Config) error {
configLk.Lock()
defer configLk.Unlock()
c.MaxTimeout = getMaxTimeout(updated.MaxTimeout)
return nil
}
// GetMaxTimeout - returns the per call drive operation timeout
func (c *Config) GetMaxTimeout() time.Duration {
return c.GetOPTimeout()
}
// GetOPTimeout - returns the per call drive operation timeout
func (c *Config) GetOPTimeout() time.Duration {
configLk.RLock()
defer configLk.RUnlock()
return getMaxTimeout(c.MaxTimeout)
}
// LookupConfig - lookup config and override with valid environment settings if any.
func LookupConfig(kvs config.KVS) (cfg Config, err error) {
cfg = Config{
MaxTimeout: 30 * time.Second,
}
if err = config.CheckValidKeys(config.DriveSubSys, kvs, DefaultKVS); err != nil {
return cfg, err
}
// if not set. Get default value from environment
d := env.Get(EnvMaxDriveTimeout, env.Get(EnvMaxDriveTimeoutLegacy, env.Get(EnvMaxDiskTimeoutLegacy, kvs.GetWithDefault(MaxTimeout, DefaultKVS))))
if d == "" {
cfg.MaxTimeout = 30 * time.Second
} else {
dur, _ := time.ParseDuration(d)
if dur < time.Second {
cfg.MaxTimeout = 30 * time.Second
} else {
cfg.MaxTimeout = getMaxTimeout(dur)
}
}
return cfg, err
}
func getMaxTimeout(t time.Duration) time.Duration {
if t > time.Second {
return t
}
// get default value
d := env.Get(EnvMaxDriveTimeoutLegacy, env.Get(EnvMaxDiskTimeoutLegacy, ""))
if d == "" {
return 30 * time.Second
}
dur, _ := time.ParseDuration(d)
if dur < time.Second {
return 30 * time.Second
}
return dur
}

35
internal/config/drive/help.go Normal file
View File

@@ -0,0 +1,35 @@
// Copyright (c) 2015-2023 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package drive
import "github.com/minio/minio/internal/config"
var (
// MaxTimeout is the max timeout for drive
MaxTimeout = "max_timeout"
// HelpDrive is help for drive
HelpDrive = config.HelpKVS{
config.HelpKV{
Key: MaxTimeout,
Type: "string",
Description: "set per call max_timeout for the drive, defaults to 30 seconds",
Optional: true,
},
}
)

161
internal/config/errors-utils.go Normal file
View File

@@ -0,0 +1,161 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package config
import (
"errors"
"fmt"
"net"
"syscall"
"github.com/minio/minio/internal/color"
)
// Err is a structure which contains all information
// to print a fatal error message in json or pretty mode
// Err implements error so we can use it anywhere
type Err struct {
msg string
detail string
action string
hint string
}
// Clone returns a new Err struct with the same information
func (u Err) Clone() Err {
return Err{
msg: u.msg,
detail: u.detail,
action: u.action,
hint: u.hint,
}
}
// Error returns the error message
func (u Err) Error() string {
if u.detail == "" {
if u.msg != "" {
return u.msg
}
return "<nil>"
}
return u.detail
}
// Msg - Replace the current error's message
func (u Err) Msg(m string) Err {
e := u.Clone()
e.msg = m
return e
}
// Msgf - Replace the current error's message
func (u Err) Msgf(m string, args ...any) Err {
e := u.Clone()
if len(args) == 0 {
e.msg = m
} else {
e.msg = fmt.Sprintf(m, args...)
}
return e
}
// Hint - Replace the current error's message
func (u Err) Hint(m string, args ...any) Err {
e := u.Clone()
e.hint = fmt.Sprintf(m, args...)
return e
}
// ErrFn function wrapper
type ErrFn func(err error) Err
// Create a UI error generator, this is needed to simplify
// the update of the detailed error message in several places
// in MinIO code
func newErrFn(msg, action, hint string) ErrFn {
return func(err error) Err {
u := Err{
msg: msg,
action: action,
hint: hint,
}
if err != nil {
u.detail = err.Error()
}
return u
}
}
// ErrorToErr inspects the passed error and transforms it
// to the appropriate UI error.
func ErrorToErr(err error) Err {
if err == nil {
return Err{}
}
// If this is already a Err, do nothing
if e, ok := err.(Err); ok {
return e
}
// Show a generic message for known golang errors
if errors.Is(err, syscall.EADDRINUSE) {
return ErrPortAlreadyInUse(err).Msg("Specified port is already in use")
} else if errors.Is(err, syscall.EACCES) || errors.Is(err, syscall.EPERM) {
if netErr, ok := err.(*net.OpError); ok {
return ErrPortAccess(netErr).Msg("Insufficient permissions to use specified port")
}
}
// Failed to identify what type of error this, return a simple UI error
return Err{msg: err.Error()}
}
// FmtError converts a fatal error message to a more clear error
// using some colors
func FmtError(introMsg string, err error, jsonFlag bool) string {
renderedTxt := ""
uiErr := ErrorToErr(err)
// JSON print
if jsonFlag {
// Message text in json should be simple
if uiErr.detail != "" {
return uiErr.msg + ": " + uiErr.detail
}
return uiErr.msg
}
// Pretty print error message
introMsg += ": "
if uiErr.msg != "" {
introMsg += color.Bold(uiErr.msg)
} else {
introMsg += color.Bold(err.Error())
}
renderedTxt += color.Red(introMsg) + "\n"
// Add action message
if uiErr.action != "" {
renderedTxt += "> " + color.BgYellow(color.Black(uiErr.action)) + "\n"
}
// Add hint
if uiErr.hint != "" {
renderedTxt += color.Bold("HINT:") + "\n"
renderedTxt += " " + uiErr.hint
}
return renderedTxt
}

248
internal/config/errors.go Normal file
View File

@@ -0,0 +1,248 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package config
// UI errors
var (
ErrInvalidXLValue = newErrFn(
"Invalid drive path",
"Please provide a fresh drive for single drive MinIO setup",
"MinIO only supports fresh drive paths",
)
ErrInvalidBrowserValue = newErrFn(
"Invalid console value",
"Please check the passed value",
"Environment can only accept `on` and `off` values. To disable Console access, set this value to `off`",
)
ErrInvalidFSOSyncValue = newErrFn(
"Invalid O_SYNC value",
"Please check the passed value",
"Can only accept `on` and `off` values. To enable O_SYNC for fs backend, set this value to `on`",
)
ErrOverlappingDomainValue = newErrFn(
"Overlapping domain values",
"Please check the passed value",
"MINIO_DOMAIN only accepts non-overlapping domain values",
)
ErrInvalidDomainValue = newErrFn(
"Invalid domain value",
"Please check the passed value",
"Domain can only accept DNS compatible values",
)
ErrInvalidErasureSetSize = newErrFn(
"Invalid erasure set size",
"Please check the passed value",
"Erasure set can only accept any of [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] values",
)
ErrInvalidWormValue = newErrFn(
"Invalid WORM value",
"Please check the passed value",
"WORM can only accept `on` and `off` values. To enable WORM, set this value to `on`",
)
ErrInvalidConfigDecryptionKey = newErrFn(
"Incorrect encryption key to decrypt internal data",
"Please set the correct default KMS key value or the correct root credentials for older MinIO versions.",
`Revert MINIO_KMS_KES_KEY_NAME or MINIO_ROOT_USER/MINIO_ROOT_PASSWORD (for older MinIO versions) to be able to decrypt the internal data again.`,
)
ErrInvalidCredentials = newErrFn(
"Invalid credentials",
"Please provide correct credentials",
`Access key length should be at least 3, and secret key length at least 8 characters`,
)
ErrInvalidRootUserCredentials = newErrFn(
"Invalid credentials",
"Please provide correct credentials",
EnvRootUser+` length should be at least 3, and `+EnvRootPassword+` length at least 8 characters`,
)
ErrMissingEnvCredentialRootUser = newErrFn(
"Missing credential environment variable, \""+EnvRootUser+"\"",
"Environment variable \""+EnvRootUser+"\" is missing",
`Root user name (access key) and root password (secret key) are expected to be specified via environment variables MINIO_ROOT_USER and MINIO_ROOT_PASSWORD respectively`,
)
ErrMissingEnvCredentialRootPassword = newErrFn(
"Missing credential environment variable, \""+EnvRootPassword+"\"",
"Environment variable \""+EnvRootPassword+"\" is missing",
`Root user name (access key) and root password (secret key) are expected to be specified via environment variables MINIO_ROOT_USER and MINIO_ROOT_PASSWORD respectively`,
)
ErrMissingEnvCredentialAccessKey = newErrFn(
"Missing credential environment variable, \""+EnvAccessKey+"\"",
"Environment variables \""+EnvAccessKey+"\" and \""+EnvSecretKey+"\" are deprecated",
`Root user name (access key) and root password (secret key) are expected to be specified via environment variables MINIO_ROOT_USER and MINIO_ROOT_PASSWORD respectively`,
)
ErrMissingEnvCredentialSecretKey = newErrFn(
"Missing credential environment variable, \""+EnvSecretKey+"\"",
"Environment variables \""+EnvSecretKey+"\" and \""+EnvAccessKey+"\" are deprecated",
`Root user name (access key) and root password (secret key) are expected to be specified via environment variables MINIO_ROOT_USER and MINIO_ROOT_PASSWORD respectively`,
)
ErrInvalidErasureEndpoints = newErrFn(
"Invalid endpoint(s) in erasure mode",
"Please provide correct combination of local/remote paths",
"For more information, please refer to https://docs.min.io/community/minio-object-store/operations/concepts/erasure-coding.html",
)
ErrInvalidNumberOfErasureEndpoints = newErrFn(
"Invalid total number of endpoints for erasure mode",
"Please provide number of endpoints greater or equal to 2",
"For more information, please refer to https://docs.min.io/community/minio-object-store/operations/concepts/erasure-coding.html",
)
ErrStorageClassValue = newErrFn(
"Invalid storage class value",
"Please check the value",
`MINIO_STORAGE_CLASS_STANDARD: Format "EC:<Default_Parity_Standard_Class>" (e.g. "EC:3"). This sets the number of parity drives for MinIO server in Standard mode. Objects are stored in Standard mode, if storage class is not defined in Put request
MINIO_STORAGE_CLASS_RRS: Format "EC:<Default_Parity_Reduced_Redundancy_Class>" (e.g. "EC:3"). This sets the number of parity drives for MinIO server in Reduced Redundancy mode. Objects are stored in Reduced Redundancy mode, if Put request specifies RRS storage class
Refer to the link https://github.com/minio/minio/tree/master/docs/erasure/storage-class for more information`,
)
ErrUnexpectedBackendVersion = newErrFn(
"Backend version seems to be too recent",
"Please update to the latest MinIO version",
"",
)
ErrInvalidAddressFlag = newErrFn(
"--address input is invalid",
"Please check --address parameter",
`--address binds to a specific ADDRESS:PORT, ADDRESS can be an IPv4/IPv6 address or hostname (default port is ':9000')
Examples: --address ':443'
--address '172.16.34.31:9000'
--address '[fe80::da00:a6c8:e3ae:ddd7]:9000'`,
)
ErrInvalidEndpoint = newErrFn(
"Invalid endpoint for single drive mode",
"Please check the endpoint",
`Single-Node modes requires absolute path without hostnames:
Examples:
$ minio server /data/minio/ #Single Node Single Drive
$ minio server /data-{1...4}/minio # Single Node Multi Drive`,
)
ErrUnsupportedBackend = newErrFn(
"Unable to write to the backend",
"Please ensure your drive supports O_DIRECT",
"",
)
ErrUnableToWriteInBackend = newErrFn(
"Unable to write to the backend",
"Please ensure MinIO binary has write permissions for the backend",
`Verify if MinIO binary is running as the same user who has write permissions for the backend`,
)
ErrPortAlreadyInUse = newErrFn(
"Port is already in use",
"Please ensure no other program uses the same address/port",
"",
)
ErrPortAccess = newErrFn(
"Unable to use specified port",
"Please ensure MinIO binary has 'cap_net_bind_service=+ep' permissions",
`Use 'sudo setcap cap_net_bind_service=+ep /path/to/minio' to provide sufficient permissions`,
)
ErrTLSReadError = newErrFn(
"Cannot read the TLS certificate",
"Please check if the certificate has the proper owner and read permissions",
"",
)
ErrTLSUnexpectedData = newErrFn(
"Invalid TLS certificate",
"Please check your certificate",
"",
)
ErrTLSNoPassword = newErrFn(
"Missing TLS password",
"Please set the password to environment variable `MINIO_CERT_PASSWD` so that the private key can be decrypted",
"",
)
ErrNoCertsAndHTTPSEndpoints = newErrFn(
"HTTPS specified in endpoints, but no TLS certificate is found on the local machine",
"Please add TLS certificate or use HTTP endpoints only",
"Refer to https://docs.min.io/community/minio-object-store/operations/network-encryption.html for information about how to load a TLS certificate in your server",
)
ErrCertsAndHTTPEndpoints = newErrFn(
"HTTP specified in endpoints, but the server in the local machine is configured with a TLS certificate",
"Please remove the certificate in the configuration directory or switch to HTTPS",
"",
)
ErrTLSWrongPassword = newErrFn(
"Unable to decrypt the private key using the provided password",
"Please set the correct password in environment variable `MINIO_CERT_PASSWD`",
"",
)
ErrUnexpectedError = newErrFn(
"Unexpected error",
"Please contact MinIO at https://slack.min.io",
"",
)
ErrInvalidCompressionIncludesValue = newErrFn(
"Invalid compression include value",
"Please check the passed value",
"Compress extensions/mime-types are delimited by `,`. For eg, MINIO_COMPRESS_MIME_TYPES=\"A,B,C\"",
)
ErrInvalidReplicationWorkersValue = newErrFn(
"Invalid value for replication workers",
"",
"MINIO_API_REPLICATION_WORKERS: should be > 0",
)
ErrInvalidTransitionWorkersValue = newErrFn(
"Invalid value for transition workers",
"",
"MINIO_API_TRANSITION_WORKERS: should be >= GOMAXPROCS/2",
)
ErrInvalidBatchKeyRotationWorkersWait = newErrFn(
"Invalid value for batch key rotation workers wait",
"Please input a non-negative duration",
"keyrotation_workers_wait should be > 0ms",
)
ErrInvalidBatchReplicationWorkersWait = newErrFn(
"Invalid value for batch replication workers wait",
"Please input a non-negative duration",
"replication_workers_wait should be > 0ms",
)
ErrInvalidBatchExpirationWorkersWait = newErrFn(
"Invalid value for batch expiration workers wait",
"Please input a non-negative duration",
"expiration_workers_wait should be > 0ms",
)
)

183
internal/config/etcd/etcd.go Normal file
View File

@@ -0,0 +1,183 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package etcd
import (
"crypto/tls"
"crypto/x509"
"strings"
"time"
"github.com/minio/minio/internal/config"
"github.com/minio/minio/internal/crypto"
"github.com/minio/pkg/v3/env"
xnet "github.com/minio/pkg/v3/net"
clientv3 "go.etcd.io/etcd/client/v3"
"go.etcd.io/etcd/client/v3/namespace"
"go.uber.org/zap"
)
const (
// Default values used while communicating with etcd.
defaultDialTimeout = 5 * time.Second
defaultDialKeepAlive = 30 * time.Second
)
// etcd environment values
const (
Endpoints = "endpoints"
PathPrefix = "path_prefix"
CoreDNSPath = "coredns_path"
ClientCert = "client_cert"
ClientCertKey = "client_cert_key"
EnvEtcdEndpoints = "MINIO_ETCD_ENDPOINTS"
EnvEtcdPathPrefix = "MINIO_ETCD_PATH_PREFIX"
EnvEtcdCoreDNSPath = "MINIO_ETCD_COREDNS_PATH"
EnvEtcdClientCert = "MINIO_ETCD_CLIENT_CERT"
EnvEtcdClientCertKey = "MINIO_ETCD_CLIENT_CERT_KEY"
)
// DefaultKVS - default KV settings for etcd.
var (
DefaultKVS = config.KVS{
config.KV{
Key: Endpoints,
Value: "",
},
config.KV{
Key: PathPrefix,
Value: "",
},
config.KV{
Key: CoreDNSPath,
Value: "/skydns",
},
config.KV{
Key: ClientCert,
Value: "",
},
config.KV{
Key: ClientCertKey,
Value: "",
},
}
)
// Config - server etcd config.
type Config struct {
Enabled bool `json:"enabled"`
PathPrefix string `json:"pathPrefix"`
CoreDNSPath string `json:"coreDNSPath"`
clientv3.Config
}
// New - initialize new etcd client.
func New(cfg Config) (*clientv3.Client, error) {
if !cfg.Enabled {
return nil, nil
}
cli, err := clientv3.New(cfg.Config)
if err != nil {
return nil, err
}
cli.KV = namespace.NewKV(cli.KV, cfg.PathPrefix)
cli.Watcher = namespace.NewWatcher(cli.Watcher, cfg.PathPrefix)
cli.Lease = namespace.NewLease(cli.Lease, cfg.PathPrefix)
return cli, nil
}
func parseEndpoints(endpoints string) ([]string, bool, error) {
etcdEndpoints := strings.Split(endpoints, config.ValueSeparator)
var etcdSecure bool
for _, endpoint := range etcdEndpoints {
u, err := xnet.ParseHTTPURL(endpoint)
if err != nil {
return nil, false, err
}
if etcdSecure && u.Scheme == "http" {
return nil, false, config.Errorf("all endpoints should be https or http: %s", endpoint)
}
// If one of the endpoint is https, we will use https directly.
etcdSecure = etcdSecure || u.Scheme == "https"
}
return etcdEndpoints, etcdSecure, nil
}
// Enabled returns if etcd is enabled.
func Enabled(kvs config.KVS) bool {
endpoints := kvs.Get(Endpoints)
return endpoints != ""
}
// LookupConfig - Initialize new etcd config.
func LookupConfig(kvs config.KVS, rootCAs *x509.CertPool) (Config, error) {
cfg := Config{}
if err := config.CheckValidKeys(config.EtcdSubSys, kvs, DefaultKVS); err != nil {
return cfg, err
}
endpoints := env.Get(EnvEtcdEndpoints, kvs.Get(Endpoints))
if endpoints == "" {
return cfg, nil
}
etcdEndpoints, etcdSecure, err := parseEndpoints(endpoints)
if err != nil {
return cfg, err
}
cfg.Enabled = true
cfg.DialTimeout = defaultDialTimeout
cfg.DialKeepAliveTime = defaultDialKeepAlive
// Disable etcd client SDK logging, etcd client
// incorrectly starts logging in unexpected data
// format.
cfg.LogConfig = &zap.Config{
Level: zap.NewAtomicLevelAt(zap.FatalLevel),
Encoding: "console",
}
cfg.Endpoints = etcdEndpoints
cfg.CoreDNSPath = env.Get(EnvEtcdCoreDNSPath, kvs.Get(CoreDNSPath))
// Default path prefix for all keys on etcd, other than CoreDNSPath.
cfg.PathPrefix = env.Get(EnvEtcdPathPrefix, kvs.Get(PathPrefix))
if etcdSecure {
cfg.TLS = &tls.Config{
RootCAs: rootCAs,
PreferServerCipherSuites: true,
MinVersion: tls.VersionTLS12,
NextProtos: []string{"http/1.1", "h2"},
ClientSessionCache: tls.NewLRUClientSessionCache(64),
CipherSuites: crypto.TLSCiphersBackwardCompatible(),
CurvePreferences: crypto.TLSCurveIDs(),
}
// This is only to support client side certificate authentication
// https://coreos.com/etcd/docs/latest/op-guide/security.html
etcdClientCertFile := env.Get(EnvEtcdClientCert, kvs.Get(ClientCert))
etcdClientCertKey := env.Get(EnvEtcdClientCertKey, kvs.Get(ClientCertKey))
if etcdClientCertFile != "" && etcdClientCertKey != "" {
cfg.TLS.GetClientCertificate = func(unused *tls.CertificateRequestInfo) (*tls.Certificate, error) {
cert, err := tls.LoadX509KeyPair(etcdClientCertFile, etcdClientCertKey)
return &cert, err
}
}
}
return cfg, nil
}

70
internal/config/etcd/etcd_test.go Normal file
View File

@@ -0,0 +1,70 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package etcd
import (
"reflect"
"testing"
)
// TestParseEndpoints - tests parseEndpoints function with valid and invalid inputs.
func TestParseEndpoints(t *testing.T) {
testCases := []struct {
s string
endpoints []string
secure bool
success bool
}{
// Invalid inputs
{"https://localhost:2379,http://localhost:2380", nil, false, false},
{",,,", nil, false, false},
{"", nil, false, false},
{"ftp://localhost:2379", nil, false, false},
{"http://localhost:2379000", nil, false, false},
// Valid inputs
{
"https://localhost:2379,https://localhost:2380",
[]string{
"https://localhost:2379", "https://localhost:2380",
},
true, true,
},
{"http://localhost:2379", []string{"http://localhost:2379"}, false, true},
}
for _, testCase := range testCases {
t.Run(testCase.s, func(t *testing.T) {
endpoints, secure, err := parseEndpoints(testCase.s)
if err != nil && testCase.success {
t.Errorf("expected to succeed but failed with %s", err)
}
if !testCase.success && err == nil {
t.Error("expected failure but succeeded instead")
}
if testCase.success {
if !reflect.DeepEqual(endpoints, testCase.endpoints) {
t.Errorf("expected %s, got %s", testCase.endpoints, endpoints)
}
if secure != testCase.secure {
t.Errorf("expected %t, got %t", testCase.secure, secure)
}
}
})
}
}

68
internal/config/etcd/help.go Normal file
View File

@@ -0,0 +1,68 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package etcd
import "github.com/minio/minio/internal/config"
// etcd config documented in default config
var (
defaultHelpPostfix = func(key string) string {
return config.DefaultHelpPostfix(DefaultKVS, key)
}
Help = config.HelpKVS{
config.HelpKV{
Key: Endpoints,
Description: `comma separated list of etcd endpoints` + defaultHelpPostfix(Endpoints),
Type: "csv",
Sensitive: true,
},
config.HelpKV{
Key: PathPrefix,
Description: `namespace prefix to isolate tenants` + defaultHelpPostfix(PathPrefix),
Optional: true,
Type: "path",
},
config.HelpKV{
Key: CoreDNSPath,
Description: `shared bucket DNS records` + defaultHelpPostfix(CoreDNSPath),
Optional: true,
Type: "path",
},
config.HelpKV{
Key: ClientCert,
Description: `client cert for mTLS authentication` + defaultHelpPostfix(ClientCert),
Optional: true,
Type: "path",
Sensitive: true,
},
config.HelpKV{
Key: ClientCertKey,
Description: `client cert key for mTLS authentication` + defaultHelpPostfix(ClientCertKey),
Optional: true,
Type: "path",
Sensitive: true,
},
config.HelpKV{
Key: config.Comment,
Description: config.DefaultComment,
Optional: true,
Type: "sentence",
},
}
)

188
internal/config/heal/heal.go Normal file
View File

@@ -0,0 +1,188 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package heal
import (
"errors"
"fmt"
"strconv"
"strings"
"sync"
"time"
"github.com/minio/minio/internal/config"
"github.com/minio/pkg/v3/env"
)
// Compression environment variables
const (
Bitrot = "bitrotscan"
Sleep = "max_sleep"
IOCount = "max_io"
DriveWorkers = "drive_workers"
EnvBitrot = "MINIO_HEAL_BITROTSCAN"
EnvSleep = "MINIO_HEAL_MAX_SLEEP"
EnvIOCount = "MINIO_HEAL_MAX_IO"
EnvDriveWorkers = "MINIO_HEAL_DRIVE_WORKERS"
)
var configMutex sync.RWMutex
// Config represents the heal settings.
type Config struct {
// Bitrot will perform bitrot scan on local disk when checking objects.
Bitrot string `json:"bitrotscan"`
// maximum sleep duration between objects to slow down heal operation.
Sleep time.Duration `json:"sleep"`
IOCount int `json:"iocount"`
DriveWorkers int `json:"drive_workers"`
// Cached value from Bitrot field
cache struct {
// -1: bitrot enabled, 0: bitrot disabled, > 0: bitrot cycle
bitrotCycle time.Duration
}
}
// BitrotScanCycle returns the configured cycle for the scanner healing
// - '-1' for not enabled
// - '0' for continuous bitrot scanning
// - '> 0' interval duration between cycles
func (opts Config) BitrotScanCycle() (d time.Duration) {
configMutex.RLock()
defer configMutex.RUnlock()
return opts.cache.bitrotCycle
}
// Clone safely the heal configuration
func (opts Config) Clone() (int, time.Duration, string) {
configMutex.RLock()
defer configMutex.RUnlock()
return opts.IOCount, opts.Sleep, opts.Bitrot
}
// GetWorkers returns the number of workers, -1 is none configured
func (opts Config) GetWorkers() int {
configMutex.RLock()
defer configMutex.RUnlock()
return opts.DriveWorkers
}
// Update updates opts with nopts
func (opts *Config) Update(nopts Config) {
configMutex.Lock()
defer configMutex.Unlock()
opts.Bitrot = nopts.Bitrot
opts.IOCount = nopts.IOCount
opts.Sleep = nopts.Sleep
opts.DriveWorkers = nopts.DriveWorkers
opts.cache.bitrotCycle, _ = parseBitrotConfig(nopts.Bitrot)
}
// DefaultKVS - default KV config for heal settings
var DefaultKVS = config.KVS{
config.KV{
Key: Bitrot,
Value: config.EnableOff,
},
config.KV{
Key: Sleep,
Value: "250ms",
},
config.KV{
Key: IOCount,
Value: "100",
},
config.KV{
Key: DriveWorkers,
Value: "",
},
}
const minimumBitrotCycleInMonths = 1
func parseBitrotConfig(s string) (time.Duration, error) {
// Try to parse as a boolean
enabled, err := config.ParseBool(s)
if err == nil {
switch enabled {
case true:
return 0, nil
case false:
return -1, nil
}
}
// Try to parse as a number of months
if !strings.HasSuffix(s, "m") {
return -1, errors.New("unknown format")
}
months, err := strconv.Atoi(strings.TrimSuffix(s, "m"))
if err != nil {
return -1, err
}
if months < minimumBitrotCycleInMonths {
return -1, fmt.Errorf("minimum bitrot cycle is %d month(s)", minimumBitrotCycleInMonths)
}
return time.Duration(months) * 30 * 24 * time.Hour, nil
}
// LookupConfig - lookup config and override with valid environment settings if any.
func LookupConfig(kvs config.KVS) (cfg Config, err error) {
if err = config.CheckValidKeys(config.HealSubSys, kvs, DefaultKVS); err != nil {
return cfg, err
}
bitrot := env.Get(EnvBitrot, kvs.GetWithDefault(Bitrot, DefaultKVS))
if _, err = parseBitrotConfig(bitrot); err != nil {
return cfg, fmt.Errorf("'heal:bitrotscan' value invalid: %w", err)
}
cfg.Bitrot = bitrot
cfg.Sleep, err = time.ParseDuration(env.Get(EnvSleep, kvs.GetWithDefault(Sleep, DefaultKVS)))
if err != nil {
return cfg, fmt.Errorf("'heal:max_sleep' value invalid: %w", err)
}
cfg.IOCount, err = strconv.Atoi(env.Get(EnvIOCount, kvs.GetWithDefault(IOCount, DefaultKVS)))
if err != nil {
return cfg, fmt.Errorf("'heal:max_io' value invalid: %w", err)
}
if ws := env.Get(EnvDriveWorkers, kvs.GetWithDefault(DriveWorkers, DefaultKVS)); ws != "" {
w, err := strconv.Atoi(ws)
if err != nil {
return cfg, fmt.Errorf("'heal:drive_workers' value invalid: %w", err)
}
if w < 1 {
return cfg, fmt.Errorf("'heal:drive_workers' value invalid: zero or negative integer unsupported")
}
cfg.DriveWorkers = w
} else {
cfg.DriveWorkers = -1
}
return cfg, nil
}

55
internal/config/heal/help.go Normal file
View File

@@ -0,0 +1,55 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package heal
import "github.com/minio/minio/internal/config"
// Help template for caching feature.
var (
defaultHelpPostfix = func(key string) string {
return config.DefaultHelpPostfix(DefaultKVS, key)
}
// Help provides help for config values
Help = config.HelpKVS{
config.HelpKV{
Key: Bitrot,
Description: `perform bitrot scan on drives when checking objects during scanner` + defaultHelpPostfix(Bitrot),
Optional: true,
Type: "on|off",
},
config.HelpKV{
Key: Sleep,
Description: `maximum sleep duration between objects to slow down heal operation` + defaultHelpPostfix(Sleep),
Optional: true,
Type: "duration",
},
config.HelpKV{
Key: IOCount,
Description: `maximum IO requests allowed between objects to slow down heal operation` + defaultHelpPostfix(IOCount),
Optional: true,
Type: "int",
},
config.HelpKV{
Key: DriveWorkers,
Description: `the number of workers per drive to heal a new disk replacement` + defaultHelpPostfix(DriveWorkers),
Optional: true,
Type: "int",
},
}
)

102
internal/config/help.go Normal file
View File

@@ -0,0 +1,102 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package config
// HelpKV - implements help messages for keys
// with value as description of the keys.
type HelpKV struct {
Key string `json:"key"`
Type string `json:"type"`
Description string `json:"description"`
Optional bool `json:"optional"`
// Indicates if the value contains sensitive info that shouldn't be exposed
// in certain apis (such as Health Diagnostics/Callhome)
Sensitive bool `json:"-"`
// Indicates if the value is a secret such as a password that shouldn't be
// exposed by the server
Secret bool `json:"-"`
// Indicates if sub-sys supports multiple targets.
MultipleTargets bool `json:"multipleTargets"`
}
// HelpKVS - implement order of keys help messages.
type HelpKVS []HelpKV
// Lookup - lookup a key from help kvs.
func (hkvs HelpKVS) Lookup(key string) (HelpKV, bool) {
for _, hkv := range hkvs {
if hkv.Key == key {
return hkv, true
}
}
return HelpKV{}, false
}
// DefaultComment used across all sub-systems.
const DefaultComment = "optionally add a comment to this setting"
// Region help is documented in default config
var (
SiteHelp = HelpKVS{
HelpKV{
Key: NameKey,
Type: "string",
Description: `name for the site e.g. "cal-rack0"`,
Optional: true,
},
HelpKV{
Key: RegionKey,
Type: "string",
Description: `name of the location of the server e.g. "us-west-1"`,
Optional: true,
},
HelpKV{
Key: Comment,
Type: "sentence",
Description: DefaultComment,
Optional: true,
},
}
RegionHelp = HelpKVS{
HelpKV{
Key: RegionName,
Type: "string",
Description: `[DEPRECATED] name of the location of the server e.g. "us-west-rack2"`,
Optional: true,
},
HelpKV{
Key: Comment,
Type: "sentence",
Description: DefaultComment,
Optional: true,
},
}
)
// DefaultHelpPostfix - Helper function to add (default: $value) messages in config help
func DefaultHelpPostfix(subsystem KVS, key string) string {
val, found := subsystem.Lookup(key)
if !found || val == "" {
return ""
}
return " (default: '" + val + "')"
}

331
internal/config/identity/ldap/config.go Normal file
View File

@@ -0,0 +1,331 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package ldap
import (
"crypto/tls"
"crypto/x509"
"errors"
"net"
"sort"
"time"
"github.com/minio/madmin-go/v3"
"github.com/minio/minio/internal/config"
"github.com/minio/minio/internal/crypto"
"github.com/minio/pkg/v3/ldap"
)
const (
defaultLDAPExpiry = time.Hour * 1
minLDAPExpiry time.Duration = 15 * time.Minute
maxLDAPExpiry time.Duration = 365 * 24 * time.Hour
)
// Config contains AD/LDAP server connectivity information.
type Config struct {
LDAP ldap.Config
stsExpiryDuration time.Duration // contains converted value
}
// Enabled returns if LDAP is enabled.
func (l *Config) Enabled() bool {
return l.LDAP.Enabled
}
// Clone returns a cloned copy of LDAP config.
func (l *Config) Clone() Config {
if l == nil {
return Config{}
}
cfg := Config{
LDAP: l.LDAP.Clone(),
stsExpiryDuration: l.stsExpiryDuration,
}
return cfg
}
// LDAP keys and envs.
const (
ServerAddr = "server_addr"
SRVRecordName = "srv_record_name"
LookupBindDN = "lookup_bind_dn"
LookupBindPassword = "lookup_bind_password"
UserDNSearchBaseDN = "user_dn_search_base_dn"
UserDNSearchFilter = "user_dn_search_filter"
UserDNAttributes = "user_dn_attributes"
GroupSearchFilter = "group_search_filter"
GroupSearchBaseDN = "group_search_base_dn"
TLSSkipVerify = "tls_skip_verify"
ServerInsecure = "server_insecure"
ServerStartTLS = "server_starttls"
EnvServerAddr = "MINIO_IDENTITY_LDAP_SERVER_ADDR"
EnvSRVRecordName = "MINIO_IDENTITY_LDAP_SRV_RECORD_NAME"
EnvTLSSkipVerify = "MINIO_IDENTITY_LDAP_TLS_SKIP_VERIFY"
EnvServerInsecure = "MINIO_IDENTITY_LDAP_SERVER_INSECURE"
EnvServerStartTLS = "MINIO_IDENTITY_LDAP_SERVER_STARTTLS"
EnvUsernameFormat = "MINIO_IDENTITY_LDAP_USERNAME_FORMAT"
EnvUserDNSearchBaseDN = "MINIO_IDENTITY_LDAP_USER_DN_SEARCH_BASE_DN"
EnvUserDNSearchFilter = "MINIO_IDENTITY_LDAP_USER_DN_SEARCH_FILTER"
EnvUserDNAttributes = "MINIO_IDENTITY_LDAP_USER_DN_ATTRIBUTES"
EnvGroupSearchFilter = "MINIO_IDENTITY_LDAP_GROUP_SEARCH_FILTER"
EnvGroupSearchBaseDN = "MINIO_IDENTITY_LDAP_GROUP_SEARCH_BASE_DN"
EnvLookupBindDN = "MINIO_IDENTITY_LDAP_LOOKUP_BIND_DN"
EnvLookupBindPassword = "MINIO_IDENTITY_LDAP_LOOKUP_BIND_PASSWORD"
)
var removedKeys = []string{
"sts_expiry",
"username_format",
"username_search_filter",
"username_search_base_dn",
"group_name_attribute",
}
// DefaultKVS - default config for LDAP config
var (
DefaultKVS = config.KVS{
config.KV{
Key: config.Enable,
Value: "",
},
config.KV{
Key: ServerAddr,
Value: "",
},
config.KV{
Key: SRVRecordName,
Value: "",
},
config.KV{
Key: UserDNSearchBaseDN,
Value: "",
},
config.KV{
Key: UserDNSearchFilter,
Value: "",
},
config.KV{
Key: UserDNAttributes,
Value: "",
},
config.KV{
Key: GroupSearchFilter,
Value: "",
},
config.KV{
Key: GroupSearchBaseDN,
Value: "",
},
config.KV{
Key: TLSSkipVerify,
Value: config.EnableOff,
},
config.KV{
Key: ServerInsecure,
Value: config.EnableOff,
},
config.KV{
Key: ServerStartTLS,
Value: config.EnableOff,
},
config.KV{
Key: LookupBindDN,
Value: "",
},
config.KV{
Key: LookupBindPassword,
Value: "",
},
}
)
// Enabled returns if LDAP config is enabled.
func Enabled(kvs config.KVS) bool {
return kvs.Get(ServerAddr) != ""
}
// Lookup - initializes LDAP config, overrides config, if any ENV values are set.
func Lookup(s config.Config, rootCAs *x509.CertPool) (l Config, err error) {
l = Config{}
// Purge all removed keys first
kvs := s[config.IdentityLDAPSubSys][config.Default]
if len(kvs) > 0 {
for _, k := range removedKeys {
kvs.Delete(k)
}
s[config.IdentityLDAPSubSys][config.Default] = kvs
}
if err := s.CheckValidKeys(config.IdentityLDAPSubSys, removedKeys); err != nil {
return l, err
}
getCfgVal := func(cfgParam string) string {
// As parameters are already validated, we skip checking
// if the config param was found.
val, _, _ := s.ResolveConfigParam(config.IdentityLDAPSubSys, config.Default, cfgParam, false)
return val
}
ldapServer := getCfgVal(ServerAddr)
if ldapServer == "" {
return l, nil
}
// Set ServerName in TLS config for proper certificate validation
host, _, err := net.SplitHostPort(ldapServer)
if err != nil {
host = ldapServer
}
l.LDAP = ldap.Config{
ServerAddr: ldapServer,
SRVRecordName: getCfgVal(SRVRecordName),
TLS: &tls.Config{
ServerName: host,
MinVersion: tls.VersionTLS12,
NextProtos: []string{"h2", "http/1.1"},
ClientSessionCache: tls.NewLRUClientSessionCache(100),
CipherSuites: crypto.TLSCiphersBackwardCompatible(), // Contains RSA key exchange
RootCAs: rootCAs,
},
}
// Parse explicitly set enable=on/off flag.
isEnableFlagExplicitlySet := false
if v := getCfgVal(config.Enable); v != "" {
isEnableFlagExplicitlySet = true
l.LDAP.Enabled, err = config.ParseBool(v)
if err != nil {
return l, err
}
}
l.stsExpiryDuration = defaultLDAPExpiry
// LDAP connection configuration
if v := getCfgVal(ServerInsecure); v != "" {
l.LDAP.ServerInsecure, err = config.ParseBool(v)
if err != nil {
return l, err
}
}
if v := getCfgVal(ServerStartTLS); v != "" {
l.LDAP.ServerStartTLS, err = config.ParseBool(v)
if err != nil {
return l, err
}
}
if v := getCfgVal(TLSSkipVerify); v != "" {
l.LDAP.TLS.InsecureSkipVerify, err = config.ParseBool(v)
if err != nil {
return l, err
}
}
// Lookup bind user configuration
l.LDAP.LookupBindDN = getCfgVal(LookupBindDN)
l.LDAP.LookupBindPassword = getCfgVal(LookupBindPassword)
// User DN search configuration
l.LDAP.UserDNSearchFilter = getCfgVal(UserDNSearchFilter)
l.LDAP.UserDNSearchBaseDistName = getCfgVal(UserDNSearchBaseDN)
l.LDAP.UserDNAttributes = getCfgVal(UserDNAttributes)
// Group search params configuration
l.LDAP.GroupSearchFilter = getCfgVal(GroupSearchFilter)
l.LDAP.GroupSearchBaseDistName = getCfgVal(GroupSearchBaseDN)
// If enable flag was not explicitly set, we treat it as implicitly set at
// this point as necessary configuration is available.
if !isEnableFlagExplicitlySet && !l.LDAP.Enabled {
l.LDAP.Enabled = true
}
// Validate and test configuration.
valResult := l.LDAP.Validate()
if !valResult.IsOk() {
// Set to false if configuration fails to validate.
l.LDAP.Enabled = false
return l, valResult
}
return l, nil
}
// GetConfigList - returns a list of LDAP configurations.
func (l *Config) GetConfigList(s config.Config) ([]madmin.IDPListItem, error) {
ldapConfigs, err := s.GetAvailableTargets(config.IdentityLDAPSubSys)
if err != nil {
return nil, err
}
// For now, ldapConfigs will only have a single entry for the default
// configuration.
var res []madmin.IDPListItem
for _, cfg := range ldapConfigs {
res = append(res, madmin.IDPListItem{
Type: "ldap",
Name: cfg,
Enabled: l.Enabled(),
})
}
return res, nil
}
// ErrProviderConfigNotFound - represents a non-existing provider error.
var ErrProviderConfigNotFound = errors.New("provider configuration not found")
// GetConfigInfo - returns config details for an LDAP configuration.
func (l *Config) GetConfigInfo(s config.Config, cfgName string) ([]madmin.IDPCfgInfo, error) {
// For now only a single LDAP config is supported.
if cfgName != madmin.Default {
return nil, ErrProviderConfigNotFound
}
kvsrcs, err := s.GetResolvedConfigParams(config.IdentityLDAPSubSys, cfgName, true)
if err != nil {
return nil, err
}
res := make([]madmin.IDPCfgInfo, 0, len(kvsrcs))
for _, kvsrc := range kvsrcs {
// skip default values.
if kvsrc.Src == config.ValueSourceDef {
continue
}
res = append(res, madmin.IDPCfgInfo{
Key: kvsrc.Key,
Value: kvsrc.Value,
IsCfg: true,
IsEnv: kvsrc.Src == config.ValueSourceEnv,
})
}
// sort the structs by the key
sort.Slice(res, func(i, j int) bool {
return res[i].Key < res[j].Key
})
return res, nil
}

112
internal/config/identity/ldap/help.go Normal file
View File

@@ -0,0 +1,112 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package ldap
import "github.com/minio/minio/internal/config"
// Help template for LDAP identity feature.
var (
defaultHelpPostfix = func(key string) string {
return config.DefaultHelpPostfix(DefaultKVS, key)
}
Help = config.HelpKVS{
config.HelpKV{
Key: ServerAddr,
Description: `AD/LDAP server address e.g. "myldap.com" or "myldapserver.com:636"` + defaultHelpPostfix(ServerAddr),
Type: "address",
Sensitive: true,
},
config.HelpKV{
Key: SRVRecordName,
Description: `DNS SRV record name for LDAP service, if given, must be one of "ldap", "ldaps" or "on"` + defaultHelpPostfix(SRVRecordName),
Optional: true,
Type: "string",
Sensitive: false,
},
config.HelpKV{
Key: LookupBindDN,
Description: `DN for LDAP read-only service account used to perform DN and group lookups` + defaultHelpPostfix(LookupBindDN),
Optional: true,
Type: "string",
Sensitive: true,
},
config.HelpKV{
Key: LookupBindPassword,
Description: `Password for LDAP read-only service account used to perform DN and group lookups` + defaultHelpPostfix(LookupBindPassword),
Optional: true,
Type: "string",
Sensitive: true,
Secret: true,
},
config.HelpKV{
Key: UserDNSearchBaseDN,
Description: `";" separated list of user search base DNs e.g. "dc=myldapserver,dc=com"` + defaultHelpPostfix(UserDNSearchBaseDN),
Optional: true,
Type: "list",
},
config.HelpKV{
Key: UserDNSearchFilter,
Description: `Search filter to lookup user DN` + defaultHelpPostfix(UserDNSearchFilter),
Optional: true,
Type: "string",
},
config.HelpKV{
Key: UserDNAttributes,
Description: `"," separated list of user DN attributes e.g. "uid,cn,mail,sshPublicKey"` + defaultHelpPostfix(UserDNAttributes),
Optional: true,
Type: "list",
},
config.HelpKV{
Key: GroupSearchFilter,
Description: `search filter for groups e.g. "(&(objectclass=groupOfNames)(memberUid=%s))"` + defaultHelpPostfix(GroupSearchFilter),
Optional: true,
Type: "string",
},
config.HelpKV{
Key: GroupSearchBaseDN,
Description: `";" separated list of group search base DNs e.g. "dc=myldapserver,dc=com"` + defaultHelpPostfix(GroupSearchBaseDN),
Optional: true,
Type: "list",
},
config.HelpKV{
Key: TLSSkipVerify,
Description: `trust server TLS without verification` + defaultHelpPostfix(TLSSkipVerify),
Optional: true,
Type: "on|off",
},
config.HelpKV{
Key: ServerInsecure,
Description: `allow plain text connection to AD/LDAP server` + defaultHelpPostfix(ServerInsecure),
Optional: true,
Type: "on|off",
},
config.HelpKV{
Key: ServerStartTLS,
Description: `use StartTLS connection to AD/LDAP server` + defaultHelpPostfix(ServerStartTLS),
Optional: true,
Type: "on|off",
},
config.HelpKV{
Key: config.Comment,
Description: config.DefaultComment,
Optional: true,
Type: "sentence",
},
}
)

422
internal/config/identity/ldap/ldap.go Normal file
View File

@@ -0,0 +1,422 @@
// Copyright (c) 2015-2022 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package ldap
import (
"errors"
"fmt"
"strconv"
"strings"
"time"
ldap "github.com/go-ldap/ldap/v3"
"github.com/minio/minio-go/v7/pkg/set"
"github.com/minio/minio/internal/auth"
xldap "github.com/minio/pkg/v3/ldap"
)
// LookupUserDN searches for the full DN and groups of a given short/login
// username.
func (l *Config) LookupUserDN(username string) (*xldap.DNSearchResult, []string, error) {
conn, err := l.LDAP.Connect()
if err != nil {
return nil, nil, err
}
defer conn.Close()
// Bind to the lookup user account
if err = l.LDAP.LookupBind(conn); err != nil {
return nil, nil, err
}
// Lookup user DN
lookupRes, err := l.LDAP.LookupUsername(conn, username)
if err != nil {
errRet := fmt.Errorf("Unable to find user DN: %w", err)
return nil, nil, errRet
}
groups, err := l.LDAP.SearchForUserGroups(conn, username, lookupRes.ActualDN)
if err != nil {
return nil, nil, err
}
return lookupRes, groups, nil
}
// GetValidatedDNForUsername checks if the given username exists in the LDAP directory.
// The given username could be just the short "login" username or the full DN.
//
// When the username/DN is found, the full DN returned by the **server** is
// returned, otherwise the returned string is empty. The value returned here is
// the value sent by the LDAP server and is used in minio as the server performs
// LDAP specific normalization (including Unicode normalization).
//
// If the user is not found, err = nil, otherwise, err != nil.
func (l *Config) GetValidatedDNForUsername(username string) (*xldap.DNSearchResult, error) {
conn, err := l.LDAP.Connect()
if err != nil {
return nil, err
}
defer conn.Close()
// Bind to the lookup user account
if err = l.LDAP.LookupBind(conn); err != nil {
return nil, err
}
// Check if the passed in username is a valid DN.
if !l.ParsesAsDN(username) {
// We consider it as a login username and attempt to check it exists in
// the directory.
bindDN, err := l.LDAP.LookupUsername(conn, username)
if err != nil {
if strings.Contains(err.Error(), "User DN not found for") {
return nil, nil
}
return nil, fmt.Errorf("Unable to find user DN: %w", err)
}
return bindDN, nil
}
// Since the username parses as a valid DN, check that it exists and is
// under a configured base DN in the LDAP directory.
validDN, isUnderBaseDN, err := l.GetValidatedUserDN(conn, username)
if err == nil && !isUnderBaseDN {
// Not under any configured base DN, so treat as not found.
return nil, nil
}
return validDN, err
}
// GetValidatedUserDN validates the given user DN. Will error out if conn is nil. The returned
// boolean is true iff the user DN is found under one of the LDAP user base DNs.
func (l *Config) GetValidatedUserDN(conn *ldap.Conn, userDN string) (*xldap.DNSearchResult, bool, error) {
return l.GetValidatedDNUnderBaseDN(conn, userDN,
l.LDAP.GetUserDNSearchBaseDistNames(), l.LDAP.GetUserDNAttributesList())
}
// GetValidatedGroupDN validates the given group DN. If conn is nil, creates a
// connection. The returned boolean is true iff the group DN is found under one
// of the configured LDAP base DNs.
func (l *Config) GetValidatedGroupDN(conn *ldap.Conn, groupDN string) (*xldap.DNSearchResult, bool, error) {
if conn == nil {
var err error
conn, err = l.LDAP.Connect()
if err != nil {
return nil, false, err
}
defer conn.Close()
// Bind to the lookup user account
if err = l.LDAP.LookupBind(conn); err != nil {
return nil, false, err
}
}
return l.GetValidatedDNUnderBaseDN(conn, groupDN,
l.LDAP.GetGroupSearchBaseDistNames(), nil)
}
// GetValidatedDNUnderBaseDN checks if the given DN exists in the LDAP
// directory.
//
// The `NormDN` value returned here in the search result may not be equal to the
// input DN, as LDAP equality is not a simple Golang string equality. However,
// we assume the value returned by the LDAP server is canonical. Additionally,
// the attribute type names in the DN are lower-cased.
//
// Return values:
//
// If the DN is found, the normalized (string) value and any requested
// attributes are returned and error is nil.
//
// If the DN is not found, a nil result and error are returned.
//
// The returned boolean is true iff the DN is found under one of the LDAP
// subtrees listed in `baseDNList`.
func (l *Config) GetValidatedDNUnderBaseDN(conn *ldap.Conn, dn string, baseDNList []xldap.BaseDNInfo, attrs []string) (*xldap.DNSearchResult, bool, error) {
if len(baseDNList) == 0 {
return nil, false, errors.New("no Base DNs given")
}
// Check that DN exists in the LDAP directory.
searchRes, err := xldap.LookupDN(conn, dn, attrs)
if err != nil {
return nil, false, fmt.Errorf("Error looking up DN %s: %w", dn, err)
}
if searchRes == nil {
return nil, false, nil
}
// This will not return an error as the argument is validated to be a DN.
pdn, _ := ldap.ParseDN(searchRes.NormDN)
// Check that the DN is under a configured base DN in the LDAP
// directory.
for _, baseDN := range baseDNList {
if baseDN.Parsed.AncestorOf(pdn) {
return searchRes, true, nil
}
}
// Not under any configured base DN so return false.
return searchRes, false, nil
}
// GetValidatedDNWithGroups - Gets validated DN from given DN or short username
// and returns the DN and the groups the user is a member of.
//
// If username is required in group search but a DN is passed, no groups are
// returned.
func (l *Config) GetValidatedDNWithGroups(username string) (*xldap.DNSearchResult, []string, error) {
conn, err := l.LDAP.Connect()
if err != nil {
return nil, nil, err
}
defer conn.Close()
// Bind to the lookup user account
if err = l.LDAP.LookupBind(conn); err != nil {
return nil, nil, err
}
var lookupRes *xldap.DNSearchResult
shortUsername := ""
// Check if the passed in username is a valid DN.
if !l.ParsesAsDN(username) {
// We consider it as a login username and attempt to check it exists in
// the directory.
lookupRes, err = l.LDAP.LookupUsername(conn, username)
if err != nil {
if strings.Contains(err.Error(), "User DN not found for") {
return nil, nil, nil
}
return nil, nil, fmt.Errorf("Unable to find user DN: %w", err)
}
shortUsername = username
} else {
// Since the username parses as a valid DN, check that it exists and is
// under a configured base DN in the LDAP directory.
var isUnderBaseDN bool
lookupRes, isUnderBaseDN, err = l.GetValidatedUserDN(conn, username)
if err == nil && !isUnderBaseDN {
return nil, nil, fmt.Errorf("Unable to find user DN: %w", err)
}
}
groups, err := l.LDAP.SearchForUserGroups(conn, shortUsername, lookupRes.ActualDN)
if err != nil {
return nil, nil, err
}
return lookupRes, groups, nil
}
// Bind - binds to ldap, searches LDAP and returns the distinguished name of the
// user and the list of groups.
func (l *Config) Bind(username, password string) (*xldap.DNSearchResult, []string, error) {
conn, err := l.LDAP.Connect()
if err != nil {
return nil, nil, err
}
defer conn.Close()
// Bind to the lookup user account
if err = l.LDAP.LookupBind(conn); err != nil {
return nil, nil, err
}
// Lookup user DN
lookupResult, err := l.LDAP.LookupUsername(conn, username)
if err != nil {
errRet := fmt.Errorf("Unable to find user DN: %w", err)
return nil, nil, errRet
}
// Authenticate the user credentials.
err = conn.Bind(lookupResult.ActualDN, password)
if err != nil {
errRet := fmt.Errorf("LDAP auth failed for DN %s: %w", lookupResult.ActualDN, err)
return nil, nil, errRet
}
// Bind to the lookup user account again to perform group search.
if err = l.LDAP.LookupBind(conn); err != nil {
return nil, nil, err
}
// User groups lookup.
groups, err := l.LDAP.SearchForUserGroups(conn, username, lookupResult.ActualDN)
if err != nil {
return nil, nil, err
}
return lookupResult, groups, nil
}
// GetExpiryDuration - return parsed expiry duration.
func (l Config) GetExpiryDuration(dsecs string) (time.Duration, error) {
if dsecs == "" {
return l.stsExpiryDuration, nil
}
d, err := strconv.Atoi(dsecs)
if err != nil {
return 0, auth.ErrInvalidDuration
}
dur := time.Duration(d) * time.Second
if dur < minLDAPExpiry || dur > maxLDAPExpiry {
return 0, auth.ErrInvalidDuration
}
return dur, nil
}
// ParsesAsDN determines if the given string could be a valid DN based on
// parsing alone.
func (l Config) ParsesAsDN(dn string) bool {
_, err := ldap.ParseDN(dn)
return err == nil
}
// IsLDAPUserDN determines if the given string could be a user DN from LDAP.
func (l Config) IsLDAPUserDN(user string) bool {
udn, err := ldap.ParseDN(user)
if err != nil {
return false
}
for _, baseDN := range l.LDAP.GetUserDNSearchBaseDistNames() {
if baseDN.Parsed.AncestorOf(udn) {
return true
}
}
return false
}
// IsLDAPGroupDN determines if the given string could be a group DN from LDAP.
func (l Config) IsLDAPGroupDN(group string) bool {
gdn, err := ldap.ParseDN(group)
if err != nil {
return false
}
for _, baseDN := range l.LDAP.GetGroupSearchBaseDistNames() {
if baseDN.Parsed.AncestorOf(gdn) {
return true
}
}
return false
}
// GetNonEligibleUserDistNames - find user accounts (DNs) that are no longer
// present in the LDAP server or do not meet filter criteria anymore
func (l *Config) GetNonEligibleUserDistNames(userDistNames []string) ([]string, error) {
conn, err := l.LDAP.Connect()
if err != nil {
return nil, err
}
defer conn.Close()
// Bind to the lookup user account
if err = l.LDAP.LookupBind(conn); err != nil {
return nil, err
}
// Evaluate the filter again with generic wildcard instead of specific values
filter := strings.ReplaceAll(l.LDAP.UserDNSearchFilter, "%s", "*")
nonExistentUsers := []string{}
for _, dn := range userDistNames {
searchRequest := ldap.NewSearchRequest(
dn,
ldap.ScopeBaseObject, ldap.NeverDerefAliases, 0, 0, false,
filter,
[]string{}, // only need DN, so pass no attributes here
nil,
)
searchResult, err := conn.Search(searchRequest)
if err != nil {
// Object does not exist error?
if ldap.IsErrorWithCode(err, 32) {
ndn, err := ldap.ParseDN(dn)
if err != nil {
return nil, err
}
nonExistentUsers = append(nonExistentUsers, ndn.String())
continue
}
return nil, err
}
if len(searchResult.Entries) == 0 {
// DN was not found - this means this user account is
// expired.
ndn, err := ldap.ParseDN(dn)
if err != nil {
return nil, err
}
nonExistentUsers = append(nonExistentUsers, ndn.String())
}
}
return nonExistentUsers, nil
}
// LookupGroupMemberships - for each DN finds the set of LDAP groups they are a
// member of.
func (l *Config) LookupGroupMemberships(userDistNames []string, userDNToUsernameMap map[string]string) (map[string]set.StringSet, error) {
conn, err := l.LDAP.Connect()
if err != nil {
return nil, err
}
defer conn.Close()
// Bind to the lookup user account
if err = l.LDAP.LookupBind(conn); err != nil {
return nil, err
}
res := make(map[string]set.StringSet, len(userDistNames))
for _, userDistName := range userDistNames {
username := userDNToUsernameMap[userDistName]
groups, err := l.LDAP.SearchForUserGroups(conn, username, userDistName)
if err != nil {
return nil, err
}
res[userDistName] = set.CreateStringSet(groups...)
}
return res, nil
}
// QuickNormalizeDN - normalizes the given DN without checking if it is valid or
// exists in the LDAP directory. Returns input if error
func (l Config) QuickNormalizeDN(dn string) string {
if normDN, err := xldap.NormalizeDN(dn); err == nil {
return normDN
}
return dn
}
// DecodeDN - denormalizes the given DN by unescaping any escaped characters.
// Returns input if error
func (l Config) DecodeDN(dn string) string {
if decodedDN, err := xldap.DecodeDN(dn); err == nil {
return decodedDN
}
return dn
}

67
internal/config/identity/ldap/legacy.go Normal file
View File

@@ -0,0 +1,67 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package ldap
import (
"github.com/minio/minio/internal/config"
)
// LegacyConfig contains AD/LDAP server connectivity information from old config
// V33.
type LegacyConfig struct {
Enabled bool `json:"enabled"`
// E.g. "ldap.minio.io:636"
ServerAddr string `json:"serverAddr"`
// User DN search parameters
UserDNSearchBaseDistName string `json:"userDNSearchBaseDN"`
UserDNSearchBaseDistNames []string `json:"-"` // Generated field
UserDNSearchFilter string `json:"userDNSearchFilter"`
// Group search parameters
GroupSearchBaseDistName string `json:"groupSearchBaseDN"`
GroupSearchBaseDistNames []string `json:"-"` // Generated field
GroupSearchFilter string `json:"groupSearchFilter"`
// Lookup bind LDAP service account
LookupBindDN string `json:"lookupBindDN"`
LookupBindPassword string `json:"lookupBindPassword"`
}
// SetIdentityLDAP - One time migration code needed, for migrating from older config to new for LDAPConfig.
func SetIdentityLDAP(s config.Config, ldapArgs LegacyConfig) {
if !ldapArgs.Enabled {
// ldap not enabled no need to preserve it in new settings.
return
}
s[config.IdentityLDAPSubSys][config.Default] = config.KVS{
config.KV{
Key: ServerAddr,
Value: ldapArgs.ServerAddr,
},
config.KV{
Key: GroupSearchFilter,
Value: ldapArgs.GroupSearchFilter,
},
config.KV{
Key: GroupSearchBaseDN,
Value: ldapArgs.GroupSearchBaseDistName,
},
}
}

50
internal/config/identity/openid/ecdsa-sha3_contrib.go Normal file
View File

@@ -0,0 +1,50 @@
// MinIO Object Storage (c) 2021 MinIO, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package openid
import (
"crypto"
"github.com/golang-jwt/jwt/v4"
// Needed for SHA3 to work - See: https://golang.org/src/crypto/crypto.go?s=1034:1288
_ "golang.org/x/crypto/sha3"
)
// Specific instances for EC256 and company
var (
SigningMethodES3256 *jwt.SigningMethodECDSA
SigningMethodES3384 *jwt.SigningMethodECDSA
SigningMethodES3512 *jwt.SigningMethodECDSA
)
func init() {
// ES256
SigningMethodES3256 = &jwt.SigningMethodECDSA{Name: "ES3256", Hash: crypto.SHA3_256, KeySize: 32, CurveBits: 256}
jwt.RegisterSigningMethod(SigningMethodES3256.Alg(), func() jwt.SigningMethod {
return SigningMethodES3256
})
// ES384
SigningMethodES3384 = &jwt.SigningMethodECDSA{Name: "ES3384", Hash: crypto.SHA3_384, KeySize: 48, CurveBits: 384}
jwt.RegisterSigningMethod(SigningMethodES3384.Alg(), func() jwt.SigningMethod {
return SigningMethodES3384
})
// ES512
SigningMethodES3512 = &jwt.SigningMethodECDSA{Name: "ES3512", Hash: crypto.SHA3_512, KeySize: 66, CurveBits: 521}
jwt.RegisterSigningMethod(SigningMethodES3512.Alg(), func() jwt.SigningMethod {
return SigningMethodES3512
})
}

119
internal/config/identity/openid/help.go Normal file
View File

@@ -0,0 +1,119 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package openid
import "github.com/minio/minio/internal/config"
// Help template for OpenID identity feature.
var (
defaultHelpPostfix = func(key string) string {
return config.DefaultHelpPostfix(DefaultKVS, key)
}
Help = config.HelpKVS{
config.HelpKV{
Key: DisplayName,
Description: "Friendly display name for this Provider/App" + defaultHelpPostfix(DisplayName),
Optional: true,
Type: "string",
},
config.HelpKV{
Key: ConfigURL,
Description: `openid discovery document e.g. "https://accounts.google.com/.well-known/openid-configuration"` + defaultHelpPostfix(ConfigURL),
Type: "url",
},
config.HelpKV{
Key: ClientID,
Description: `unique public identifier for apps e.g. "292085223830.apps.googleusercontent.com"` + defaultHelpPostfix(ClientID),
Type: "string",
},
config.HelpKV{
Key: ClientSecret,
Description: `secret for the unique public identifier for apps` + defaultHelpPostfix(ClientSecret),
Sensitive: true,
Type: "string",
Secret: true,
},
config.HelpKV{
Key: RolePolicy,
Description: `Set the IAM access policies applicable to this client application and IDP e.g. "app-bucket-write,app-bucket-list"` + defaultHelpPostfix(RolePolicy),
Optional: true,
Type: "string",
},
config.HelpKV{
Key: ClaimName,
Description: `JWT canned policy claim name` + defaultHelpPostfix(ClaimName),
Optional: true,
Type: "string",
},
config.HelpKV{
Key: Scopes,
Description: `Comma separated list of OpenID scopes for server, defaults to advertised scopes from discovery document e.g. "email,admin"` + defaultHelpPostfix(Scopes),
Optional: true,
Type: "csv",
},
config.HelpKV{
Key: Vendor,
Description: `Specify vendor type for vendor specific behavior to checking validity of temporary credentials and service accounts on MinIO` + defaultHelpPostfix(Vendor),
Optional: true,
Type: "string",
},
config.HelpKV{
Key: ClaimUserinfo,
Description: `Enable fetching claims from UserInfo Endpoint for authenticated user` + defaultHelpPostfix(ClaimUserinfo),
Optional: true,
Type: "on|off",
},
config.HelpKV{
Key: KeyCloakRealm,
Description: `Specify Keycloak 'realm' name, only honored if vendor was set to 'keycloak' as value, if no realm is specified 'master' is default` + defaultHelpPostfix(KeyCloakRealm),
Optional: true,
Type: "string",
},
config.HelpKV{
Key: KeyCloakAdminURL,
Description: `Specify Keycloak 'admin' REST API endpoint e.g. http://localhost:8080/auth/admin/` + defaultHelpPostfix(KeyCloakAdminURL),
Optional: true,
Type: "string",
},
config.HelpKV{
Key: RedirectURIDynamic,
Description: `Enable 'Host' header based dynamic redirect URI` + defaultHelpPostfix(RedirectURIDynamic),
Optional: true,
Type: "on|off",
},
config.HelpKV{
Key: ClaimPrefix,
Description: `[DEPRECATED use 'claim_name'] JWT claim namespace prefix e.g. "customer1/"` + defaultHelpPostfix(ClaimPrefix),
Optional: true,
Type: "string",
},
config.HelpKV{
Key: RedirectURI,
Description: `[DEPRECATED use env 'MINIO_BROWSER_REDIRECT_URL'] Configure custom redirect_uri for OpenID login flow callback` + defaultHelpPostfix(RedirectURI),
Optional: true,
Type: "string",
},
config.HelpKV{
Key: config.Comment,
Description: config.DefaultComment,
Optional: true,
Type: "sentence",
},
}
)

130
internal/config/identity/openid/jwks.go Normal file
View File

@@ -0,0 +1,130 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package openid
import (
"crypto"
"crypto/ecdsa"
"crypto/ed25519"
"crypto/elliptic"
"crypto/rsa"
"encoding/base64"
"errors"
"fmt"
"math/big"
)
// JWKS - https://tools.ietf.org/html/rfc7517
type JWKS struct {
Keys []*JWKS `json:"keys,omitempty"`
Kty string `json:"kty"`
Use string `json:"use,omitempty"`
Kid string `json:"kid,omitempty"`
Alg string `json:"alg,omitempty"`
Crv string `json:"crv,omitempty"`
X string `json:"x,omitempty"`
Y string `json:"y,omitempty"`
D string `json:"d,omitempty"`
N string `json:"n,omitempty"`
E string `json:"e,omitempty"`
K string `json:"k,omitempty"`
}
var (
errMalformedJWKRSAKey = errors.New("malformed JWK RSA key")
errMalformedJWKECKey = errors.New("malformed JWK EC key")
)
// DecodePublicKey - decodes JSON Web Key (JWK) as public key
func (key *JWKS) DecodePublicKey() (crypto.PublicKey, error) {
switch key.Kty {
case "RSA":
if key.N == "" || key.E == "" {
return nil, errMalformedJWKRSAKey
}
// decode exponent
ebuf, err := base64.RawURLEncoding.DecodeString(key.E)
if err != nil {
return nil, errMalformedJWKRSAKey
}
nbuf, err := base64.RawURLEncoding.DecodeString(key.N)
if err != nil {
return nil, errMalformedJWKRSAKey
}
var n, e big.Int
n.SetBytes(nbuf)
e.SetBytes(ebuf)
return &rsa.PublicKey{
E: int(e.Int64()),
N: &n,
}, nil
case "EC":
if key.Crv == "" || key.X == "" || key.Y == "" {
return nil, errMalformedJWKECKey
}
var curve elliptic.Curve
switch key.Crv {
case "P-224":
curve = elliptic.P224()
case "P-256":
curve = elliptic.P256()
case "P-384":
curve = elliptic.P384()
case "P-521":
curve = elliptic.P521()
default:
return nil, fmt.Errorf("Unknown curve type: %s", key.Crv)
}
xbuf, err := base64.RawURLEncoding.DecodeString(key.X)
if err != nil {
return nil, errMalformedJWKECKey
}
ybuf, err := base64.RawURLEncoding.DecodeString(key.Y)
if err != nil {
return nil, errMalformedJWKECKey
}
var x, y big.Int
x.SetBytes(xbuf)
y.SetBytes(ybuf)
return &ecdsa.PublicKey{
Curve: curve,
X: &x,
Y: &y,
}, nil
default:
if key.Alg == "EdDSA" && key.Crv == "Ed25519" && key.X != "" {
pb, err := base64.RawURLEncoding.DecodeString(key.X)
if err != nil {
return nil, errMalformedJWKECKey
}
return ed25519.PublicKey(pb), nil
}
return nil, fmt.Errorf("Unknown JWK key type %s", key.Kty)
}
}

134
internal/config/identity/openid/jwks_test.go Normal file
View File

@@ -0,0 +1,134 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package openid
import (
"bytes"
"crypto"
"crypto/ecdsa"
"crypto/elliptic"
"crypto/rsa"
"encoding/json"
"testing"
)
func TestAzurePublicKey(t *testing.T) {
const jsonkey = `{"keys":[{"kty":"RSA","use":"sig","kid":"SsZsBNhZcF3Q9S4trpQBTByNRRI","x5t":"SsZsBNhZcF3Q9S4trpQBTByNRRI","n":"uHPewhg4WC3eLVPkEFlj7RDtaKYWXCI5G-LPVzsMKOuIu7qQQbeytIA6P6HT9_iIRt8zNQvuw4P9vbNjgUCpI6vfZGsjk3XuCVoB_bAIhvuBcQh9ePH2yEwS5reR-NrG1PsqzobnZZuigKCoDmuOb_UDx1DiVyNCbMBlEG7UzTQwLf5NP6HaRHx027URJeZvPAWY7zjHlSOuKoS_d1yUveaBFIgZqPWLCg44ck4gvik45HsNVWT9zYfT74dvUSSrMSR-SHFT7Hy1XjbVXpHJHNNAXpPoGoWXTuc0BxMsB4cqjfJqoftFGOG4x32vEzakArLPxAKwGvkvu0jToAyvSQ","e":"AQAB","x5c":"MIIDBTCCAe2gAwIBAgIQWHw7h/Ysh6hPcXpnrJ0N8DANBgkqhkiG9w0BAQsFADAtMSswKQYDVQQDEyJhY2NvdW50cy5hY2Nlc3Njb250cm9sLndpbmRvd3MubmV0MB4XDTIwMDQyNzAwMDAwMFoXDTI1MDQyNzAwMDAwMFowLTErMCkGA1UEAxMiYWNjb3VudHMuYWNjZXNzY29udHJvbC53aW5kb3dzLm5ldDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALhz3sIYOFgt3i1T5BBZY+0Q7WimFlwiORviz1c7DCjriLu6kEG3srSAOj+h0/f4iEbfMzUL7sOD/b2zY4FAqSOr32RrI5N17glaAf2wCIb7gXEIfXjx9shMEua3kfjaxtT7Ks6G52WbooCgqA5rjm/1A8dQ4lcjQmzAZRBu1M00MC3+TT+h2kR8dNu1ESXmbzwFmO84x5UjriqEv3dclL3mgRSIGaj1iwoOOHJOIL4pOOR7DVVk/c2H0++Hb1EkqzEkfkhxU+x8tV421V6RyRzTQF6T6BqFl07nNAcTLAeHKo3yaqH7RRjhuMd9rxM2pAKyz8QCsBr5L7tI06AMr0kCAwEAAaMhMB8wHQYDVR0OBBYEFOI7M+DDFMlP7Ac3aomPnWo1QL1SMA0GCSqGSIb3DQEBCwUAA4IBAQBv+8rBiDY8sZDBoUDYwFQM74QjqCmgNQfv5B0Vjwg20HinERjQeH24uAWzyhWN9++FmeY4zcRXDY5UNmB0nJz7UGlprA9s7voQ0Lkyiud0DO072RPBg38LmmrqoBsLb3MB9MZ2CGBaHftUHfpdTvrgmXSP0IJn7mCUq27g+hFk7n/MLbN1k8JswEODIgdMRvGqN+mnrPKkviWmcVAZccsWfcmS1pKwXqICTKzd6WmVdz+cL7ZSd9I2X0pY4oRwauoE2bS95vrXljCYgLArI3XB2QcnglDDBRYu3Z3aIJb26PTIyhkVKT7xaXhXl4OgrbmQon9/O61G2dzpjzzBPqNP","issuer":"https://login.microsoftonline.com/906aefe9-76a7-4f65-b82d-5ec20775d5aa/v2.0"},{"kty":"RSA","use":"sig","kid":"huN95IvPfehq34GzBDZ1GXGirnM","x5t":"huN95IvPfehq34GzBDZ1GXGirnM","n":"6lldKm5Rc_vMKa1RM_TtUv3tmtj52wLRrJqu13yGM3_h0dwru2ZP53y65wDfz6_tLCjoYuRCuVsjoW37-0zXUORJvZ0L90CAX-58lW7NcE4bAzA1pXv7oR9kQw0X8dp0atU4HnHeaTU8LZxcjJO79_H9cxgwa-clKfGxllcos8TsuurM8xi2dx5VqwzqNMB2s62l3MTN7AzctHUiQCiX2iJArGjAhs-mxS1wmyMIyOSipdodhjQWRAcseW-aFVyRTFVi8okl2cT1HJjPXdx0b1WqYSOzeRdrrLUcA0oR2Tzp7xzOYJZSGNnNLQqa9f6h6h52XbX0iAgxKgEDlRpbJw","e":"AQAB","x5c":["MIIDBTCCAe2gAwIBAgIQPCxFbySVSLZOggeWRzBWOjANBgkqhkiG9w0BAQsFADAtMSswKQYDVQQDEyJhY2NvdW50cy5hY2Nlc3Njb250cm9sLndpbmRvd3MubmV0MB4XDTIwMDYwNzAwMDAwMFoXDTI1MDYwNzAwMDAwMFowLTErMCkGA1UEAxMiYWNjb3VudHMuYWNjZXNzY29udHJvbC53aW5kb3dzLm5ldDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAOpZXSpuUXP7zCmtUTP07VL97ZrY+dsC0ayartd8hjN/4dHcK7tmT+d8uucA38+v7Swo6GLkQrlbI6Ft+/tM11DkSb2dC/dAgF/ufJVuzXBOGwMwNaV7+6EfZEMNF/HadGrVOB5x3mk1PC2cXIyTu/fx/XMYMGvnJSnxsZZXKLPE7LrqzPMYtnceVasM6jTAdrOtpdzEzewM3LR1IkAol9oiQKxowIbPpsUtcJsjCMjkoqXaHYY0FkQHLHlvmhVckUxVYvKJJdnE9RyYz13cdG9VqmEjs3kXa6y1HANKEdk86e8czmCWUhjZzS0KmvX+oeoedl219IgIMSoBA5UaWycCAwEAAaMhMB8wHQYDVR0OBBYEFFXP0ODFhjf3RS6oRijM5Tb+yB8CMA0GCSqGSIb3DQEBCwUAA4IBAQB9GtVikLTbJWIu5x9YCUTTKzNhi44XXogP/v8VylRSUHI5YTMdnWwvDIt/Y1sjNonmSy9PrioEjcIiI1U8nicveafMwIq5VLn+gEY2lg6KDJAzgAvA88CXqwfHHvtmYBovN7goolp8TY/kddMTf6TpNzN3lCTM2MK4Ye5xLLVGdp4bqWCOJ/qjwDxpTRSydYIkLUDwqNjv+sYfOElJpYAB4rTL/aw3ChJ1iaA4MtXEt6OjbUtbOa21lShfLzvNRbYK3+ukbrhmRl9lemJEeUls51vPuIe+jg+Ssp43aw7PQjxt4/MpfNMS2BfZ5F8GVSVG7qNb352cLLeJg5rc398Z"],"issuer":"https://login.microsoftonline.com/906aefe9-76a7-4f65-b82d-5ec20775d5aa/v2.0"},{"kty":"RSA","use":"sig","kid":"M6pX7RHoraLsprfJeRCjSxuURhc","x5t":"M6pX7RHoraLsprfJeRCjSxuURhc","n":"xHScZMPo8FifoDcrgncWQ7mGJtiKhrsho0-uFPXg-OdnRKYudTD7-Bq1MDjcqWRf3IfDVjFJixQS61M7wm9wALDj--lLuJJ9jDUAWTA3xWvQLbiBM-gqU0sj4mc2lWm6nPfqlyYeWtQcSC0sYkLlayNgX4noKDaXivhVOp7bwGXq77MRzeL4-9qrRYKjuzHfZL7kNBCsqO185P0NI2Jtmw-EsqYsrCaHsfNRGRrTvUHUq3hWa859kK_5uNd7TeY2ZEwKVD8ezCmSfR59ZzyxTtuPpkCSHS9OtUvS3mqTYit73qcvprjl3R8hpjXLb8oftfpWr3hFRdpxrwuoQEO4QQ","e":"AQAB","x5c":["MIIC8TCCAdmgAwIBAgIQfEWlTVc1uINEc9RBi6qHMjANBgkqhkiG9w0BAQsFADAjMSEwHwYDVQQDExhsb2dpbi5taWNyb3NvZnRvbmxpbmUudXMwHhcNMTgxMDE0MDAwMDAwWhcNMjAxMDE0MDAwMDAwWjAjMSEwHwYDVQQDExhsb2dpbi5taWNyb3NvZnRvbmxpbmUudXMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDEdJxkw+jwWJ+gNyuCdxZDuYYm2IqGuyGjT64U9eD452dEpi51MPv4GrUwONypZF/ch8NWMUmLFBLrUzvCb3AAsOP76Uu4kn2MNQBZMDfFa9AtuIEz6CpTSyPiZzaVabqc9+qXJh5a1BxILSxiQuVrI2BfiegoNpeK+FU6ntvAZervsxHN4vj72qtFgqO7Md9kvuQ0EKyo7Xzk/Q0jYm2bD4SypiysJoex81EZGtO9QdSreFZrzn2Qr/m413tN5jZkTApUPx7MKZJ9Hn1nPLFO24+mQJIdL061S9LeapNiK3vepy+muOXdHyGmNctvyh+1+laveEVF2nGvC6hAQ7hBAgMBAAGjITAfMB0GA1UdDgQWBBQ5TKadw06O0cvXrQbXW0Nb3M3h/DANBgkqhkiG9w0BAQsFAAOCAQEAI48JaFtwOFcYS/3pfS5+7cINrafXAKTL+/+he4q+RMx4TCu/L1dl9zS5W1BeJNO2GUznfI+b5KndrxdlB6qJIDf6TRHh6EqfA18oJP5NOiKhU4pgkF2UMUw4kjxaZ5fQrSoD9omjfHAFNjradnHA7GOAoF4iotvXDWDBWx9K4XNZHWvD11Td66zTg5IaEQDIZ+f8WS6nn/98nAVMDtR9zW7Te5h9kGJGfe6WiHVaGRPpBvqC4iypGHjbRwANwofZvmp5wP08hY1CsnKY5tfP+E2k/iAQgKKa6QoxXToYvP7rsSkglak8N5g/+FJGnq4wP6cOzgZpjdPMwaVt5432GA=="],"issuer":"https://login.microsoftonline.com/906aefe9-76a7-4f65-b82d-5ec20775d5aa/v2.0"}]}`
var jk JWKS
if err := json.Unmarshal([]byte(jsonkey), &jk); err != nil {
t.Fatal("Unmarshal: ", err)
} else if len(jk.Keys) != 3 {
t.Fatalf("Expected 3 keys, got %d", len(jk.Keys))
}
var kids []string
for ii, jks := range jk.Keys {
_, err := jks.DecodePublicKey()
if err != nil {
t.Fatalf("Failed to decode key %d: %v", ii, err)
}
kids = append(kids, jks.Kid)
}
if len(kids) != 3 {
t.Fatalf("Failed to find the expected number of kids: 3, got %d", len(kids))
}
}
// A.1 - Example public keys
func TestPublicKey(t *testing.T) {
const jsonkey = `{"keys":
[
{"kty":"EC",
"crv":"P-256",
"x":"MKBCTNIcKUSDii11ySs3526iDZ8AiTo7Tu6KPAqv7D4",
"y":"4Etl6SRW2YiLUrN5vfvVHuhp7x8PxltmWWlbbM4IFyM",
"use":"enc",
"kid":"1"},
{"kty":"RSA",
"n": "0vx7agoebGcQSuuPiLJXZptN9nndrQmbXEps2aiAFbWhM78LhWx4cbbfAAtVT86zwu1RK7aPFFxuhDR1L6tSoc_BJECPebWKRXjBZCiFV4n3oknjhMstn64tZ_2W-5JsGY4Hc5n9yBXArwl93lqt7_RN5w6Cf0h4QyQ5v-65YGjQR0_FDW2QvzqY368QQMicAtaSqzs8KJZgnYb9c7d0zgdAZHzu6qMQvRL5hajrn1n91CbOpbISD08qNLyrdkt-bFTWhAI4vMQFh6WeZu0fM4lFd2NcRwr3XPksINHaQ-G_xBniIqbw0Ls1jF44-csFCur-kEgU8awapJzKnqDKgw",
"e":"AQAB",
"alg":"RS256",
"kid":"2011-04-29"}
]
}`
var jk JWKS
if err := json.Unmarshal([]byte(jsonkey), &jk); err != nil {
t.Fatal("Unmarshal: ", err)
} else if len(jk.Keys) != 2 {
t.Fatalf("Expected 2 keys, got %d", len(jk.Keys))
}
keys := make([]crypto.PublicKey, len(jk.Keys))
for ii, jks := range jk.Keys {
var err error
keys[ii], err = jks.DecodePublicKey()
if err != nil {
t.Fatalf("Failed to decode key %d: %v", ii, err)
}
}
//nolint:gocritic
if key0, ok := keys[0].(*ecdsa.PublicKey); !ok {
t.Fatalf("Expected ECDSA key[0], got %T", keys[0])
} else if key1, ok := keys[1].(*rsa.PublicKey); !ok {
t.Fatalf("Expected RSA key[1], got %T", keys[1])
} else if key0.Curve != elliptic.P256() {
t.Fatal("Key[0] is not using P-256 curve")
} else if !bytes.Equal(key0.X.Bytes(), []byte{
0x30, 0xa0, 0x42, 0x4c, 0xd2,
0x1c, 0x29, 0x44, 0x83, 0x8a, 0x2d, 0x75, 0xc9, 0x2b, 0x37, 0xe7, 0x6e, 0xa2,
0xd, 0x9f, 0x0, 0x89, 0x3a, 0x3b, 0x4e, 0xee, 0x8a, 0x3c, 0xa, 0xaf, 0xec, 0x3e,
}) {
t.Fatalf("Bad key[0].X, got %v", key0.X.Bytes())
} else if !bytes.Equal(key0.Y.Bytes(), []byte{
0xe0, 0x4b, 0x65, 0xe9, 0x24,
0x56, 0xd9, 0x88, 0x8b, 0x52, 0xb3, 0x79, 0xbd, 0xfb, 0xd5, 0x1e, 0xe8,
0x69, 0xef, 0x1f, 0xf, 0xc6, 0x5b, 0x66, 0x59, 0x69, 0x5b, 0x6c, 0xce,
0x8, 0x17, 0x23,
}) {
t.Fatalf("Bad key[0].Y, got %v", key0.Y.Bytes())
} else if key1.E != 0x10001 {
t.Fatalf("Bad key[1].E: %d", key1.E)
} else if !bytes.Equal(key1.N.Bytes(), []byte{
0xd2, 0xfc, 0x7b, 0x6a, 0xa, 0x1e,
0x6c, 0x67, 0x10, 0x4a, 0xeb, 0x8f, 0x88, 0xb2, 0x57, 0x66, 0x9b, 0x4d, 0xf6,
0x79, 0xdd, 0xad, 0x9, 0x9b, 0x5c, 0x4a, 0x6c, 0xd9, 0xa8, 0x80, 0x15, 0xb5,
0xa1, 0x33, 0xbf, 0xb, 0x85, 0x6c, 0x78, 0x71, 0xb6, 0xdf, 0x0, 0xb, 0x55,
0x4f, 0xce, 0xb3, 0xc2, 0xed, 0x51, 0x2b, 0xb6, 0x8f, 0x14, 0x5c, 0x6e, 0x84,
0x34, 0x75, 0x2f, 0xab, 0x52, 0xa1, 0xcf, 0xc1, 0x24, 0x40, 0x8f, 0x79, 0xb5,
0x8a, 0x45, 0x78, 0xc1, 0x64, 0x28, 0x85, 0x57, 0x89, 0xf7, 0xa2, 0x49, 0xe3,
0x84, 0xcb, 0x2d, 0x9f, 0xae, 0x2d, 0x67, 0xfd, 0x96, 0xfb, 0x92, 0x6c, 0x19,
0x8e, 0x7, 0x73, 0x99, 0xfd, 0xc8, 0x15, 0xc0, 0xaf, 0x9, 0x7d, 0xde, 0x5a,
0xad, 0xef, 0xf4, 0x4d, 0xe7, 0xe, 0x82, 0x7f, 0x48, 0x78, 0x43, 0x24, 0x39,
0xbf, 0xee, 0xb9, 0x60, 0x68, 0xd0, 0x47, 0x4f, 0xc5, 0xd, 0x6d, 0x90, 0xbf,
0x3a, 0x98, 0xdf, 0xaf, 0x10, 0x40, 0xc8, 0x9c, 0x2, 0xd6, 0x92, 0xab, 0x3b,
0x3c, 0x28, 0x96, 0x60, 0x9d, 0x86, 0xfd, 0x73, 0xb7, 0x74, 0xce, 0x7, 0x40,
0x64, 0x7c, 0xee, 0xea, 0xa3, 0x10, 0xbd, 0x12, 0xf9, 0x85, 0xa8, 0xeb, 0x9f,
0x59, 0xfd, 0xd4, 0x26, 0xce, 0xa5, 0xb2, 0x12, 0xf, 0x4f, 0x2a, 0x34, 0xbc,
0xab, 0x76, 0x4b, 0x7e, 0x6c, 0x54, 0xd6, 0x84, 0x2, 0x38, 0xbc, 0xc4, 0x5, 0x87,
0xa5, 0x9e, 0x66, 0xed, 0x1f, 0x33, 0x89, 0x45, 0x77, 0x63, 0x5c, 0x47, 0xa,
0xf7, 0x5c, 0xf9, 0x2c, 0x20, 0xd1, 0xda, 0x43, 0xe1, 0xbf, 0xc4, 0x19, 0xe2,
0x22, 0xa6, 0xf0, 0xd0, 0xbb, 0x35, 0x8c, 0x5e, 0x38, 0xf9, 0xcb, 0x5, 0xa, 0xea,
0xfe, 0x90, 0x48, 0x14, 0xf1, 0xac, 0x1a, 0xa4, 0x9c, 0xca, 0x9e, 0xa0, 0xca, 0x83,
}) {
t.Fatalf("Bad key[1].N, got %v", key1.N.Bytes())
}
}

286
internal/config/identity/openid/jwt.go Normal file
View File

@@ -0,0 +1,286 @@
// Copyright (c) 2015-2022 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package openid
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"sync"
"time"
jwtgo "github.com/golang-jwt/jwt/v4"
"github.com/minio/minio/internal/arn"
"github.com/minio/minio/internal/auth"
xnet "github.com/minio/pkg/v3/net"
"github.com/minio/pkg/v3/policy"
)
type publicKeys struct {
*sync.RWMutex
// map of kid to public key
pkMap map[string]any
}
func (pk *publicKeys) parseAndAdd(b io.Reader) error {
var jwk JWKS
err := json.NewDecoder(b).Decode(&jwk)
if err != nil {
return err
}
for _, key := range jwk.Keys {
pkey, err := key.DecodePublicKey()
if err != nil {
return err
}
pk.add(key.Kid, pkey)
}
return nil
}
func (pk *publicKeys) add(keyID string, key any) {
pk.Lock()
defer pk.Unlock()
pk.pkMap[keyID] = key
}
func (pk *publicKeys) get(kid string) any {
pk.RLock()
defer pk.RUnlock()
return pk.pkMap[kid]
}
// PopulatePublicKey - populates a new publickey from the JWKS URL.
func (r *Config) PopulatePublicKey(arn arn.ARN) error {
pCfg := r.arnProviderCfgsMap[arn]
if pCfg.JWKS.URL == nil || pCfg.JWKS.URL.String() == "" {
return nil
}
// Add client secret for the client ID for HMAC based signature.
r.pubKeys.add(pCfg.ClientID, []byte(pCfg.ClientSecret))
client := &http.Client{
Transport: r.transport,
}
resp, err := client.Get(pCfg.JWKS.URL.String())
if err != nil {
return err
}
defer r.closeRespFn(resp.Body)
if resp.StatusCode != http.StatusOK {
return errors.New(resp.Status)
}
return r.pubKeys.parseAndAdd(resp.Body)
}
// ErrTokenExpired - error token expired
var (
ErrTokenExpired = errors.New("token expired")
)
func updateClaimsExpiry(dsecs string, claims map[string]any) error {
expStr := claims["exp"]
if expStr == "" {
return ErrTokenExpired
}
// No custom duration requested, the claims can be used as is.
if dsecs == "" {
return nil
}
if _, err := auth.ExpToInt64(expStr); err != nil {
return err
}
defaultExpiryDuration, err := GetDefaultExpiration(dsecs)
if err != nil {
return err
}
claims["exp"] = time.Now().UTC().Add(defaultExpiryDuration).Unix() // update with new expiry.
return nil
}
const (
audClaim = "aud"
azpClaim = "azp"
)
// Validate - validates the id_token.
func (r *Config) Validate(ctx context.Context, arn arn.ARN, token, accessToken, dsecs string, claims map[string]any) error {
jp := new(jwtgo.Parser)
jp.ValidMethods = []string{
"RS256", "RS384", "RS512",
"ES256", "ES384", "ES512",
"HS256", "HS384", "HS512",
"RS3256", "RS3384", "RS3512",
"ES3256", "ES3384", "ES3512",
}
keyFuncCallback := func(jwtToken *jwtgo.Token) (any, error) {
kid, ok := jwtToken.Header["kid"].(string)
if !ok {
return nil, fmt.Errorf("Invalid kid value %v", jwtToken.Header["kid"])
}
pubkey := r.pubKeys.get(kid)
if pubkey == nil {
return nil, fmt.Errorf("No public key found for kid %s", kid)
}
return pubkey, nil
}
pCfg, ok := r.arnProviderCfgsMap[arn]
if !ok {
return fmt.Errorf("Role %s does not exist", arn)
}
mclaims := jwtgo.MapClaims(claims)
jwtToken, err := jp.ParseWithClaims(token, &mclaims, keyFuncCallback)
if err != nil {
// Re-populate the public key in-case the JWKS
// pubkeys are refreshed
if err = r.PopulatePublicKey(arn); err != nil {
return err
}
jwtToken, err = jwtgo.ParseWithClaims(token, &mclaims, keyFuncCallback)
if err != nil {
return err
}
}
if !jwtToken.Valid {
return ErrTokenExpired
}
if err = updateClaimsExpiry(dsecs, mclaims); err != nil {
return err
}
if err = r.updateUserinfoClaims(ctx, arn, accessToken, mclaims); err != nil {
return err
}
// Validate that matching clientID appears in the aud or azp claims.
// REQUIRED. Audience(s) that this ID Token is intended for.
// It MUST contain the OAuth 2.0 client_id of the Relying Party
// as an audience value. It MAY also contain identifiers for
// other audiences. In the general case, the aud value is an
// array of case sensitive strings. In the common special case
// when there is one audience, the aud value MAY be a single
// case sensitive
audValues, ok := policy.GetValuesFromClaims(mclaims, audClaim)
if !ok {
return errors.New("STS JWT Token has `aud` claim invalid, `aud` must match configured OpenID Client ID")
}
if !audValues.Contains(pCfg.ClientID) {
// if audience claims is missing, look for "azp" claims.
// OPTIONAL. Authorized party - the party to which the ID
// Token was issued. If present, it MUST contain the OAuth
// 2.0 Client ID of this party. This Claim is only needed
// when the ID Token has a single audience value and that
// audience is different than the authorized party. It MAY
// be included even when the authorized party is the same
// as the sole audience. The azp value is a case sensitive
// string containing a StringOrURI value
azpValues, ok := policy.GetValuesFromClaims(mclaims, azpClaim)
if !ok {
return errors.New("STS JWT Token has `azp` claim invalid, `azp` must match configured OpenID Client ID")
}
if !azpValues.Contains(pCfg.ClientID) {
return errors.New("STS JWT Token has `azp` claim invalid, `azp` must match configured OpenID Client ID")
}
}
return nil
}
func (r *Config) updateUserinfoClaims(ctx context.Context, arn arn.ARN, accessToken string, claims map[string]any) error {
pCfg, ok := r.arnProviderCfgsMap[arn]
// If claim user info is enabled, get claims from userInfo
// and overwrite them with the claims from JWT.
if ok && pCfg.ClaimUserinfo {
if accessToken == "" {
return errors.New("access_token is mandatory if user_info claim is enabled")
}
uclaims, err := pCfg.UserInfo(ctx, accessToken, r.transport)
if err != nil {
return err
}
for k, v := range uclaims {
if _, ok := claims[k]; !ok { // only add to claims not update it.
claims[k] = v
}
}
}
return nil
}
// DiscoveryDoc - parses the output from openid-configuration
// for example https://accounts.google.com/.well-known/openid-configuration
type DiscoveryDoc struct {
Issuer string `json:"issuer,omitempty"`
AuthEndpoint string `json:"authorization_endpoint,omitempty"`
TokenEndpoint string `json:"token_endpoint,omitempty"`
EndSessionEndpoint string `json:"end_session_endpoint,omitempty"`
UserInfoEndpoint string `json:"userinfo_endpoint,omitempty"`
RevocationEndpoint string `json:"revocation_endpoint,omitempty"`
JwksURI string `json:"jwks_uri,omitempty"`
ResponseTypesSupported []string `json:"response_types_supported,omitempty"`
SubjectTypesSupported []string `json:"subject_types_supported,omitempty"`
IDTokenSigningAlgValuesSupported []string `json:"id_token_signing_alg_values_supported,omitempty"`
ScopesSupported []string `json:"scopes_supported,omitempty"`
TokenEndpointAuthMethods []string `json:"token_endpoint_auth_methods_supported,omitempty"`
ClaimsSupported []string `json:"claims_supported,omitempty"`
CodeChallengeMethodsSupported []string `json:"code_challenge_methods_supported,omitempty"`
}
func parseDiscoveryDoc(u *xnet.URL, transport http.RoundTripper, closeRespFn func(io.ReadCloser)) (DiscoveryDoc, error) {
d := DiscoveryDoc{}
req, err := http.NewRequest(http.MethodGet, u.String(), nil)
if err != nil {
return d, err
}
clnt := http.Client{
Transport: transport,
}
resp, err := clnt.Do(req)
if err != nil {
return d, err
}
defer closeRespFn(resp.Body)
if resp.StatusCode != http.StatusOK {
return d, fmt.Errorf("unexpected error returned by %s : status(%s)", u, resp.Status)
}
dec := json.NewDecoder(resp.Body)
if err = dec.Decode(&d); err != nil {
return d, err
}
return d, nil
}

300
internal/config/identity/openid/jwt_test.go Normal file
View File

@@ -0,0 +1,300 @@
// Copyright (c) 2015-2022 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package openid
import (
"bytes"
"encoding/base64"
"encoding/json"
"fmt"
"io"
"net/http"
"net/http/httptest"
"net/url"
"sync"
"testing"
"time"
jwtgo "github.com/golang-jwt/jwt/v4"
"github.com/minio/minio/internal/arn"
"github.com/minio/minio/internal/config"
jwtm "github.com/minio/minio/internal/jwt"
xnet "github.com/minio/pkg/v3/net"
)
func TestUpdateClaimsExpiry(t *testing.T) {
testCases := []struct {
exp any
dsecs string
expectedFailure bool
}{
{"", "", true},
{"-1", "0", true},
{"-1", "900", true},
{"1574812326", "900", false},
{1574812326, "900", false},
{int64(1574812326), "900", false},
{int(1574812326), "900", false},
{uint(1574812326), "900", false},
{uint64(1574812326), "900", false},
{json.Number("1574812326"), "900", false},
{1574812326.000, "900", false},
{time.Duration(3) * time.Minute, "900", false},
}
for _, testCase := range testCases {
t.Run("", func(t *testing.T) {
claims := map[string]any{}
claims["exp"] = testCase.exp
err := updateClaimsExpiry(testCase.dsecs, claims)
if err != nil && !testCase.expectedFailure {
t.Errorf("Expected success, got failure %s", err)
}
if err == nil && testCase.expectedFailure {
t.Error("Expected failure, got success")
}
})
}
}
func initJWKSServer() *httptest.Server {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
const jsonkey = `{"keys":
[
{"kty":"RSA",
"n": "0vx7agoebGcQSuuPiLJXZptN9nndrQmbXEps2aiAFbWhM78LhWx4cbbfAAtVT86zwu1RK7aPFFxuhDR1L6tSoc_BJECPebWKRXjBZCiFV4n3oknjhMstn64tZ_2W-5JsGY4Hc5n9yBXArwl93lqt7_RN5w6Cf0h4QyQ5v-65YGjQR0_FDW2QvzqY368QQMicAtaSqzs8KJZgnYb9c7d0zgdAZHzu6qMQvRL5hajrn1n91CbOpbISD08qNLyrdkt-bFTWhAI4vMQFh6WeZu0fM4lFd2NcRwr3XPksINHaQ-G_xBniIqbw0Ls1jF44-csFCur-kEgU8awapJzKnqDKgw",
"e":"AQAB",
"alg":"RS256",
"kid":"2011-04-29"}
]
}`
w.Write([]byte(jsonkey))
}))
return server
}
func TestJWTHMACType(t *testing.T) {
server := initJWKSServer()
defer server.Close()
jwt := &jwtgo.Token{
Method: jwtgo.SigningMethodHS256,
Claims: jwtgo.StandardClaims{
ExpiresAt: 253428928061,
Audience: "76b95ae5-33ef-4283-97b7-d2a85dc2d8f4",
},
Header: map[string]any{
"typ": "JWT",
"alg": jwtgo.SigningMethodHS256.Alg(),
"kid": "76b95ae5-33ef-4283-97b7-d2a85dc2d8f4",
},
}
token, err := jwt.SignedString([]byte("WNGvKVyyNmXq0TraSvjaDN9CtpFgx35IXtGEffMCPR0"))
if err != nil {
t.Fatal(err)
}
fmt.Println(token)
u1, err := xnet.ParseHTTPURL(server.URL)
if err != nil {
t.Fatal(err)
}
pubKeys := publicKeys{
RWMutex: &sync.RWMutex{},
pkMap: map[string]any{},
}
pubKeys.add("76b95ae5-33ef-4283-97b7-d2a85dc2d8f4", []byte("WNGvKVyyNmXq0TraSvjaDN9CtpFgx35IXtGEffMCPR0"))
if len(pubKeys.pkMap) != 1 {
t.Fatalf("Expected 1 keys, got %d", len(pubKeys.pkMap))
}
provider := providerCfg{
ClientID: "76b95ae5-33ef-4283-97b7-d2a85dc2d8f4",
ClientSecret: "WNGvKVyyNmXq0TraSvjaDN9CtpFgx35IXtGEffMCPR0",
}
provider.JWKS.URL = u1
cfg := Config{
Enabled: true,
pubKeys: pubKeys,
arnProviderCfgsMap: map[arn.ARN]*providerCfg{
DummyRoleARN: &provider,
},
ProviderCfgs: map[string]*providerCfg{
"1": &provider,
},
closeRespFn: func(rc io.ReadCloser) {
rc.Close()
},
}
var claims jwtgo.MapClaims
if err = cfg.Validate(t.Context(), DummyRoleARN, token, "", "", claims); err != nil {
t.Fatal(err)
}
}
func TestJWT(t *testing.T) {
const jsonkey = `{"keys":
[
{"kty":"RSA",
"n": "0vx7agoebGcQSuuPiLJXZptN9nndrQmbXEps2aiAFbWhM78LhWx4cbbfAAtVT86zwu1RK7aPFFxuhDR1L6tSoc_BJECPebWKRXjBZCiFV4n3oknjhMstn64tZ_2W-5JsGY4Hc5n9yBXArwl93lqt7_RN5w6Cf0h4QyQ5v-65YGjQR0_FDW2QvzqY368QQMicAtaSqzs8KJZgnYb9c7d0zgdAZHzu6qMQvRL5hajrn1n91CbOpbISD08qNLyrdkt-bFTWhAI4vMQFh6WeZu0fM4lFd2NcRwr3XPksINHaQ-G_xBniIqbw0Ls1jF44-csFCur-kEgU8awapJzKnqDKgw",
"e":"AQAB",
"alg":"RS256",
"kid":"2011-04-29"}
]
}`
pubKeys := publicKeys{
RWMutex: &sync.RWMutex{},
pkMap: map[string]any{},
}
err := pubKeys.parseAndAdd(bytes.NewBuffer([]byte(jsonkey)))
if err != nil {
t.Fatal("Error loading pubkeys:", err)
}
if len(pubKeys.pkMap) != 1 {
t.Fatalf("Expected 1 keys, got %d", len(pubKeys.pkMap))
}
u1, err := xnet.ParseHTTPURL("http://127.0.0.1:8443")
if err != nil {
t.Fatal(err)
}
provider := providerCfg{}
provider.JWKS.URL = u1
cfg := Config{
Enabled: true,
pubKeys: pubKeys,
arnProviderCfgsMap: map[arn.ARN]*providerCfg{
DummyRoleARN: &provider,
},
ProviderCfgs: map[string]*providerCfg{
"1": &provider,
},
}
u, err := url.Parse("http://127.0.0.1:8443/?Token=invalid")
if err != nil {
t.Fatal(err)
}
var claims jwtgo.MapClaims
if err = cfg.Validate(t.Context(), DummyRoleARN, u.Query().Get("Token"), "", "", claims); err == nil {
t.Fatal(err)
}
}
func TestDefaultExpiryDuration(t *testing.T) {
testCases := []struct {
reqURL string
duration time.Duration
expectErr bool
}{
{
reqURL: "http://127.0.0.1:8443/?Token=xxxxx",
duration: time.Duration(60) * time.Minute,
},
{
reqURL: "http://127.0.0.1:8443/?DurationSeconds=9s",
expectErr: true,
},
{
reqURL: "http://127.0.0.1:8443/?DurationSeconds=31536001",
expectErr: true,
},
{
reqURL: "http://127.0.0.1:8443/?DurationSeconds=800",
expectErr: true,
},
{
reqURL: "http://127.0.0.1:8443/?DurationSeconds=901",
duration: time.Duration(901) * time.Second,
},
}
for i, testCase := range testCases {
u, err := url.Parse(testCase.reqURL)
if err != nil {
t.Fatal(err)
}
d, err := GetDefaultExpiration(u.Query().Get("DurationSeconds"))
gotErr := (err != nil)
if testCase.expectErr != gotErr {
t.Errorf("Test %d: Expected %v, got %v with error %s", i+1, testCase.expectErr, gotErr, err)
}
if d != testCase.duration {
t.Errorf("Test %d: Expected duration %d, got %d", i+1, testCase.duration, d)
}
}
}
func TestExpCorrect(t *testing.T) {
signKey, _ := base64.StdEncoding.DecodeString("NTNv7j0TuYARvmNMmWXo6fKvM4o6nv/aUi9ryX38ZH+L1bkrnD1ObOQ8JAUmHCBq7Iy7otZcyAagBLHVKvvYaIpmMuxmARQ97jUVG16Jkpkp1wXOPsrF9zwew6TpczyHkHgX5EuLg2MeBuiT/qJACs1J0apruOOJCg/gOtkjB4c=")
claimsMap := jwtm.NewMapClaims()
claimsMap.SetExpiry(time.Now().Add(time.Minute))
claimsMap.SetAccessKey("test-access")
if err := updateClaimsExpiry("3600", claimsMap.MapClaims); err != nil {
t.Error(err)
}
// Build simple token with updated expiration claim
token := jwtgo.NewWithClaims(jwtgo.SigningMethodHS256, claimsMap)
tokenString, err := token.SignedString(signKey)
if err != nil {
t.Error(err)
}
// Parse token to be sure it is valid
err = jwtm.ParseWithClaims(tokenString, claimsMap, func(*jwtm.MapClaims) ([]byte, error) {
return signKey, nil
})
if err != nil {
t.Error(err)
}
}
func TestKeycloakProviderInitialization(t *testing.T) {
testConfig := providerCfg{
DiscoveryDoc: DiscoveryDoc{
TokenEndpoint: "http://keycloak.test/token/endpoint",
},
}
testKvs := config.KVS{}
testKvs.Set(Vendor, "keycloak")
testKvs.Set(KeyCloakRealm, "TestRealm")
testKvs.Set(KeyCloakAdminURL, "http://keycloak.test/auth/admin")
cfgGet := func(param string) string {
return testKvs.Get(param)
}
if testConfig.provider != nil {
t.Errorf("Empty config cannot have any provider!")
}
if err := testConfig.initializeProvider(cfgGet, http.DefaultTransport); err != nil {
t.Error(err)
}
if testConfig.provider == nil {
t.Errorf("keycloak provider must be initialized!")
}
}

652
internal/config/identity/openid/openid.go Normal file
View File

@@ -0,0 +1,652 @@
// Copyright (c) 2015-2022 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package openid
import (
"crypto/sha1"
"encoding/base64"
"errors"
"io"
"maps"
"net/http"
"slices"
"sort"
"strconv"
"strings"
"sync"
"time"
"github.com/minio/madmin-go/v3"
"github.com/minio/minio-go/v7/pkg/set"
"github.com/minio/minio/internal/arn"
"github.com/minio/minio/internal/auth"
"github.com/minio/minio/internal/config"
"github.com/minio/minio/internal/config/identity/openid/provider"
"github.com/minio/minio/internal/hash/sha256"
"github.com/minio/pkg/v3/env"
xnet "github.com/minio/pkg/v3/net"
"github.com/minio/pkg/v3/policy"
)
// OpenID keys and envs.
const (
ClientID = "client_id"
ClientSecret = "client_secret"
ConfigURL = "config_url"
ClaimName = "claim_name"
ClaimUserinfo = "claim_userinfo"
RolePolicy = "role_policy"
DisplayName = "display_name"
UserReadableClaim = "user_readable_claim"
UserIDClaim = "user_id_claim"
Scopes = "scopes"
RedirectURI = "redirect_uri"
RedirectURIDynamic = "redirect_uri_dynamic"
Vendor = "vendor"
// Vendor specific ENV only enabled if the Vendor matches == "vendor"
KeyCloakRealm = "keycloak_realm"
KeyCloakAdminURL = "keycloak_admin_url"
// Removed params
JwksURL = "jwks_url"
ClaimPrefix = "claim_prefix"
)
// DefaultKVS - default config for OpenID config
var (
DefaultKVS = config.KVS{
config.KV{
Key: config.Enable,
Value: "",
},
config.KV{
Key: DisplayName,
Value: "",
},
config.KV{
Key: ConfigURL,
Value: "",
},
config.KV{
Key: ClientID,
Value: "",
},
config.KV{
Key: ClientSecret,
Value: "",
},
config.KV{
Key: ClaimName,
Value: policy.PolicyName,
},
config.KV{
Key: ClaimUserinfo,
Value: "",
},
config.KV{
Key: RolePolicy,
Value: "",
},
config.KV{
Key: ClaimPrefix,
Value: "",
HiddenIfEmpty: true,
},
config.KV{
Key: RedirectURI,
Value: "",
HiddenIfEmpty: true,
},
config.KV{
Key: RedirectURIDynamic,
Value: "off",
},
config.KV{
Key: Scopes,
Value: "",
},
config.KV{
Key: Vendor,
Value: "",
},
config.KV{
Key: KeyCloakRealm,
Value: "",
},
config.KV{
Key: KeyCloakAdminURL,
Value: "",
},
config.KV{
Key: UserReadableClaim,
Value: "",
},
config.KV{
Key: UserIDClaim,
Value: "",
},
}
)
var errSingleProvider = config.Errorf("Only one OpenID provider can be configured if not using role policy mapping")
// DummyRoleARN is used to indicate that the user associated with it was
// authenticated via policy-claim based OpenID provider.
var DummyRoleARN = func() arn.ARN {
v, err := arn.NewIAMRoleARN("dummy-internal", "")
if err != nil {
panic("should not happen!")
}
return v
}()
// Config - OpenID Config
type Config struct {
Enabled bool
// map of roleARN to providerCfg's
arnProviderCfgsMap map[arn.ARN]*providerCfg
// map of config names to providerCfg's
ProviderCfgs map[string]*providerCfg
pubKeys publicKeys
roleArnPolicyMap map[arn.ARN]string
transport http.RoundTripper
closeRespFn func(io.ReadCloser)
}
// Clone returns a cloned copy of OpenID config.
func (r *Config) Clone() Config {
if r == nil {
return Config{}
}
cfg := Config{
Enabled: r.Enabled,
arnProviderCfgsMap: make(map[arn.ARN]*providerCfg, len(r.arnProviderCfgsMap)),
ProviderCfgs: make(map[string]*providerCfg, len(r.ProviderCfgs)),
pubKeys: r.pubKeys,
roleArnPolicyMap: make(map[arn.ARN]string, len(r.roleArnPolicyMap)),
transport: r.transport,
closeRespFn: r.closeRespFn,
}
maps.Copy(cfg.arnProviderCfgsMap, r.arnProviderCfgsMap)
maps.Copy(cfg.ProviderCfgs, r.ProviderCfgs)
maps.Copy(cfg.roleArnPolicyMap, r.roleArnPolicyMap)
return cfg
}
// LookupConfig lookup jwks from config, override with any ENVs.
func LookupConfig(s config.Config, transport http.RoundTripper, closeRespFn func(io.ReadCloser), serverRegion string) (c Config, err error) {
openIDClientTransport := http.DefaultTransport
if transport != nil {
openIDClientTransport = transport
}
c = Config{
Enabled: false,
arnProviderCfgsMap: map[arn.ARN]*providerCfg{},
ProviderCfgs: map[string]*providerCfg{},
pubKeys: publicKeys{
RWMutex: &sync.RWMutex{},
pkMap: map[string]any{},
},
roleArnPolicyMap: map[arn.ARN]string{},
transport: openIDClientTransport,
closeRespFn: closeRespFn,
}
seenClientIDs := set.NewStringSet()
deprecatedKeys := []string{JwksURL}
// remove this since we have removed support for this already.
for k := range s[config.IdentityOpenIDSubSys] {
for _, dk := range deprecatedKeys {
kvs := s[config.IdentityOpenIDSubSys][k]
kvs.Delete(dk)
s[config.IdentityOpenIDSubSys][k] = kvs
}
}
if err := s.CheckValidKeys(config.IdentityOpenIDSubSys, deprecatedKeys); err != nil {
return c, err
}
openIDTargets, err := s.GetAvailableTargets(config.IdentityOpenIDSubSys)
if err != nil {
return c, err
}
for _, cfgName := range openIDTargets {
getCfgVal := func(cfgParam string) string {
// As parameters are already validated, we skip checking
// if the config param was found.
val, _, _ := s.ResolveConfigParam(config.IdentityOpenIDSubSys, cfgName, cfgParam, false)
return val
}
// In the past, when only one openID provider was allowed, there
// was no `enable` parameter - the configuration is turned off
// by clearing the values. With multiple providers, we support
// individually enabling/disabling provider configurations. If
// the enable parameter's value is non-empty, we use that
// setting, otherwise we treat it as enabled if some important
// parameters are non-empty.
var (
cfgEnableVal = getCfgVal(config.Enable)
isExplicitlyEnabled = cfgEnableVal != ""
)
var enabled bool
if isExplicitlyEnabled {
enabled, err = config.ParseBool(cfgEnableVal)
if err != nil {
return c, err
}
// No need to continue loading if the config is not enabled.
if !enabled {
continue
}
}
p := newProviderCfgFromConfig(getCfgVal)
configURL := getCfgVal(ConfigURL)
if !isExplicitlyEnabled {
enabled = true
if p.ClientID == "" && p.ClientSecret == "" && configURL == "" {
enabled = false
}
}
// No need to continue loading if the config is not enabled.
if !enabled {
continue
}
// Validate that client ID has not been duplicately specified.
if seenClientIDs.Contains(p.ClientID) {
return c, config.Errorf("Client ID %s is present with multiple OpenID configurations", p.ClientID)
}
seenClientIDs.Add(p.ClientID)
p.URL, err = xnet.ParseHTTPURL(configURL)
if err != nil {
return c, err
}
configURLDomain := p.URL.Hostname()
p.DiscoveryDoc, err = parseDiscoveryDoc(p.URL, transport, closeRespFn)
if err != nil {
return c, err
}
if p.ClaimUserinfo && configURL == "" {
return c, errors.New("please specify config_url to enable fetching claims from UserInfo endpoint")
}
if scopeList := getCfgVal(Scopes); scopeList != "" {
var scopes []string
for scope := range strings.SplitSeq(scopeList, ",") {
scope = strings.TrimSpace(scope)
if scope == "" {
return c, config.Errorf("empty scope value is not allowed '%s', please refer to our documentation", scopeList)
}
scopes = append(scopes, scope)
}
// Replace the discovery document scopes by client customized scopes.
p.DiscoveryDoc.ScopesSupported = scopes
}
// Check if claim name is the non-default value and role policy is set.
if p.ClaimName != policy.PolicyName && p.RolePolicy != "" {
// In the unlikely event that the user specifies
// `policy.PolicyName` as the claim name explicitly and sets
// a role policy, this check is thwarted, but we will be using
// the role policy anyway.
return c, config.Errorf("Role Policy (=`%s`) and Claim Name (=`%s`) cannot both be set", p.RolePolicy, p.ClaimName)
}
jwksURL := p.DiscoveryDoc.JwksURI
if jwksURL == "" {
return c, config.Errorf("no JWKS URI found in your provider's discovery doc (config_url=%s)", configURL)
}
p.JWKS.URL, err = xnet.ParseHTTPURL(jwksURL)
if err != nil {
return c, err
}
if p.RolePolicy != "" {
// RolePolicy is validated by IAM System during its
// initialization.
// Generate role ARN as combination of provider domain and
// prefix of client ID.
domain := configURLDomain
if domain == "" {
// Attempt to parse the JWKs URI.
domain = p.JWKS.URL.Hostname()
if domain == "" {
return c, config.Errorf("unable to parse a domain from the OpenID config")
}
}
if p.ClientID == "" {
return c, config.Errorf("client ID must not be empty")
}
// We set the resource ID of the role arn as a hash of client
// ID, so we can get a short roleARN that stays the same on
// restart.
var resourceID string
{
h := sha1.New()
h.Write([]byte(p.ClientID))
bs := h.Sum(nil)
resourceID = base64.RawURLEncoding.EncodeToString(bs)
}
p.roleArn, err = arn.NewIAMRoleARN(resourceID, serverRegion)
if err != nil {
return c, config.Errorf("unable to generate ARN from the OpenID config: %v", err)
}
c.roleArnPolicyMap[p.roleArn] = p.RolePolicy
} else if p.ClaimName == "" {
return c, config.Errorf("A role policy or claim name must be specified")
}
if err = p.initializeProvider(getCfgVal, c.transport); err != nil {
return c, err
}
arnKey := p.roleArn
if p.RolePolicy == "" {
arnKey = DummyRoleARN
// Ensure that at most one JWT policy claim based provider may be
// defined.
if _, ok := c.arnProviderCfgsMap[DummyRoleARN]; ok {
return c, errSingleProvider
}
}
c.arnProviderCfgsMap[arnKey] = &p
c.ProviderCfgs[cfgName] = &p
if err = c.PopulatePublicKey(arnKey); err != nil {
return c, err
}
}
c.Enabled = true
return c, nil
}
// ErrProviderConfigNotFound - represents a non-existing provider error.
var ErrProviderConfigNotFound = errors.New("provider configuration not found")
// GetConfigInfo - returns configuration and related info for the given IDP
// provider.
func (r *Config) GetConfigInfo(s config.Config, cfgName string) ([]madmin.IDPCfgInfo, error) {
openIDConfigs, err := s.GetAvailableTargets(config.IdentityOpenIDSubSys)
if err != nil {
return nil, err
}
present := slices.Contains(openIDConfigs, cfgName)
if !present {
return nil, ErrProviderConfigNotFound
}
kvsrcs, err := s.GetResolvedConfigParams(config.IdentityOpenIDSubSys, cfgName, true)
if err != nil {
return nil, err
}
res := make([]madmin.IDPCfgInfo, 0, len(kvsrcs)+1)
for _, kvsrc := range kvsrcs {
// skip returning default config values.
if kvsrc.Src == config.ValueSourceDef {
if kvsrc.Key != madmin.EnableKey {
continue
}
// for EnableKey we set an explicit on/off from live configuration
// if it is present.
if _, ok := r.ProviderCfgs[cfgName]; !ok {
// No live config is present
continue
}
if r.Enabled {
kvsrc.Value = "on"
} else {
kvsrc.Value = "off"
}
}
res = append(res, madmin.IDPCfgInfo{
Key: kvsrc.Key,
Value: kvsrc.Value,
IsCfg: true,
IsEnv: kvsrc.Src == config.ValueSourceEnv,
})
}
if provCfg, exists := r.ProviderCfgs[cfgName]; exists && provCfg.RolePolicy != "" {
// Append roleARN
res = append(res, madmin.IDPCfgInfo{
Key: "roleARN",
Value: provCfg.roleArn.String(),
IsCfg: false,
})
}
// sort the structs by the key
sort.Slice(res, func(i, j int) bool {
return res[i].Key < res[j].Key
})
return res, nil
}
// GetConfigList - list openID configurations
func (r *Config) GetConfigList(s config.Config) ([]madmin.IDPListItem, error) {
openIDConfigs, err := s.GetAvailableTargets(config.IdentityOpenIDSubSys)
if err != nil {
return nil, err
}
var res []madmin.IDPListItem
for _, cfg := range openIDConfigs {
pcfg, ok := r.ProviderCfgs[cfg]
if !ok {
res = append(res, madmin.IDPListItem{
Type: "openid",
Name: cfg,
Enabled: false,
})
} else {
var roleARN string
if pcfg.RolePolicy != "" {
roleARN = pcfg.roleArn.String()
}
res = append(res, madmin.IDPListItem{
Type: "openid",
Name: cfg,
Enabled: r.Enabled,
RoleARN: roleARN,
})
}
}
return res, nil
}
// Enabled returns if configURL is enabled.
func Enabled(kvs config.KVS) bool {
return kvs.Get(ConfigURL) != ""
}
// GetSettings - fetches OIDC settings for site-replication related validation.
// NOTE that region must be populated by caller as this package does not know.
func (r *Config) GetSettings() madmin.OpenIDSettings {
res := madmin.OpenIDSettings{}
if !r.Enabled {
return res
}
h := sha256.New()
hashedSecret := ""
for arn, provCfg := range r.arnProviderCfgsMap {
h.Write([]byte(provCfg.ClientSecret))
hashedSecret = base64.RawURLEncoding.EncodeToString(h.Sum(nil))
h.Reset()
if arn != DummyRoleARN {
if res.Roles == nil {
res.Roles = make(map[string]madmin.OpenIDProviderSettings)
}
res.Roles[arn.String()] = madmin.OpenIDProviderSettings{
ClaimUserinfoEnabled: provCfg.ClaimUserinfo,
RolePolicy: provCfg.RolePolicy,
ClientID: provCfg.ClientID,
HashedClientSecret: hashedSecret,
}
} else {
res.ClaimProvider = madmin.OpenIDProviderSettings{
ClaimUserinfoEnabled: provCfg.ClaimUserinfo,
RolePolicy: provCfg.RolePolicy,
ClientID: provCfg.ClientID,
HashedClientSecret: hashedSecret,
}
}
}
return res
}
// GetIAMPolicyClaimName - returns the policy claim name for the (at most one)
// provider configured without a role policy.
func (r *Config) GetIAMPolicyClaimName() string {
pCfg, ok := r.arnProviderCfgsMap[DummyRoleARN]
if !ok {
return ""
}
return pCfg.ClaimPrefix + pCfg.ClaimName
}
// LookupUser lookup userid for the provider
func (r Config) LookupUser(roleArn, userid string) (provider.User, error) {
// Can safely ignore error here as empty or invalid ARNs will not be
// mapped.
arnVal, _ := arn.Parse(roleArn)
pCfg, ok := r.arnProviderCfgsMap[arnVal]
if ok {
user, err := pCfg.provider.LookupUser(userid)
if err != nil && err != provider.ErrAccessTokenExpired {
return user, err
}
if err == provider.ErrAccessTokenExpired {
if err = pCfg.provider.LoginWithClientID(pCfg.ClientID, pCfg.ClientSecret); err != nil {
return user, err
}
user, err = pCfg.provider.LookupUser(userid)
}
return user, err
}
// Without any specific logic for a provider, all accounts
// are always enabled.
return provider.User{ID: userid, Enabled: true}, nil
}
// ProviderEnabled returns true if any vendor specific provider is enabled.
func (r Config) ProviderEnabled() bool {
if !r.Enabled {
return false
}
for _, v := range r.arnProviderCfgsMap {
if v.provider != nil {
return true
}
}
return false
}
// GetRoleInfo - returns ARN to policies map if a role policy based openID
// provider is configured. Otherwise returns nil.
func (r Config) GetRoleInfo() map[arn.ARN]string {
for _, p := range r.arnProviderCfgsMap {
if p.RolePolicy != "" {
return r.roleArnPolicyMap
}
}
return nil
}
// GetDefaultExpiration - returns the expiration seconds expected.
func GetDefaultExpiration(dsecs string) (time.Duration, error) {
timeout := env.Get(config.EnvMinioStsDuration, "")
defaultExpiryDuration, err := time.ParseDuration(timeout)
if err != nil {
defaultExpiryDuration = time.Hour
}
if timeout == "" && dsecs != "" {
expirySecs, err := strconv.ParseInt(dsecs, 10, 64)
if err != nil {
return 0, auth.ErrInvalidDuration
}
// The duration, in seconds, of the role session.
// The value can range from 900 seconds (15 minutes)
// up to 365 days.
if expirySecs < config.MinExpiration || expirySecs > config.MaxExpiration {
return 0, auth.ErrInvalidDuration
}
defaultExpiryDuration = time.Duration(expirySecs) * time.Second
} else if timeout == "" && dsecs == "" {
return time.Hour, nil
}
if defaultExpiryDuration.Seconds() < config.MinExpiration || defaultExpiryDuration.Seconds() > config.MaxExpiration {
return 0, auth.ErrInvalidDuration
}
return defaultExpiryDuration, nil
}
// GetUserReadableClaim returns the human readable claim name for the given
// configuration name.
func (r Config) GetUserReadableClaim(cfgName string) string {
pCfg, ok := r.ProviderCfgs[cfgName]
if ok {
return pCfg.UserReadableClaim
}
return ""
}
// GetUserIDClaim returns the user ID claim for the given configuration name, or "sub" if not set.
func (r Config) GetUserIDClaim(cfgName string) string {
pCfg, ok := r.ProviderCfgs[cfgName]
if ok {
if pCfg.UserIDClaim != "" {
return pCfg.UserIDClaim
}
return "sub"
}
return "" // an incorrect config should be handled outside this function
}

186
internal/config/identity/openid/provider/keycloak.go Normal file
View File

@@ -0,0 +1,186 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package provider
import (
"encoding/json"
"errors"
"fmt"
"net/http"
"net/url"
"path"
"strings"
"sync"
)
// Token - parses the output from IDP id_token.
type Token struct {
AccessToken string `json:"access_token"`
Expiry int `json:"expires_in"`
}
// KeycloakProvider implements Provider interface for KeyCloak Identity Provider.
type KeycloakProvider struct {
sync.Mutex
oeConfig DiscoveryDoc
client http.Client
adminURL string
realm string
// internal value refreshed
accessToken Token
}
// LoginWithUser authenticates username/password, not needed for Keycloak
func (k *KeycloakProvider) LoginWithUser(username, password string) error {
return ErrNotImplemented
}
// LoginWithClientID is implemented by Keycloak service account support
func (k *KeycloakProvider) LoginWithClientID(clientID, clientSecret string) error {
values := url.Values{}
values.Set("client_id", clientID)
values.Set("client_secret", clientSecret)
values.Set("grant_type", "client_credentials")
req, err := http.NewRequest(http.MethodPost, k.oeConfig.TokenEndpoint, strings.NewReader(values.Encode()))
if err != nil {
return err
}
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
resp, err := k.client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
var accessToken Token
if err = json.NewDecoder(resp.Body).Decode(&accessToken); err != nil {
return err
}
k.Lock()
k.accessToken = accessToken
k.Unlock()
return nil
}
// LookupUser lookup user by their userid.
func (k *KeycloakProvider) LookupUser(userid string) (User, error) {
req, err := http.NewRequest(http.MethodGet, k.adminURL, nil)
if err != nil {
return User{}, err
}
req.URL.Path = path.Join(req.URL.Path, "realms", k.realm, "users", userid)
k.Lock()
accessToken := k.accessToken
k.Unlock()
if accessToken.AccessToken == "" {
return User{}, ErrAccessTokenExpired
}
req.Header.Set("Authorization", "Bearer "+accessToken.AccessToken)
resp, err := k.client.Do(req)
if err != nil {
return User{}, err
}
defer resp.Body.Close()
switch resp.StatusCode {
case http.StatusOK, http.StatusPartialContent:
var u User
if err = json.NewDecoder(resp.Body).Decode(&u); err != nil {
return User{}, err
}
return u, nil
case http.StatusNotFound:
return User{
ID: userid,
Enabled: false,
}, nil
case http.StatusUnauthorized:
return User{}, ErrAccessTokenExpired
}
return User{}, fmt.Errorf("Unable to lookup - keycloak user lookup returned %v", resp.Status)
}
// Option is a function type that accepts a pointer Target
type Option func(*KeycloakProvider)
// WithTransport provide custom transport
func WithTransport(transport http.RoundTripper) Option {
return func(p *KeycloakProvider) {
p.client = http.Client{
Transport: transport,
}
}
}
// WithOpenIDConfig provide OpenID Endpoint configuration discovery document
func WithOpenIDConfig(oeConfig DiscoveryDoc) Option {
return func(p *KeycloakProvider) {
p.oeConfig = oeConfig
}
}
// WithAdminURL provide admin URL configuration for Keycloak
func WithAdminURL(url string) Option {
return func(p *KeycloakProvider) {
p.adminURL = url
}
}
// WithRealm provide realm configuration for Keycloak
func WithRealm(realm string) Option {
return func(p *KeycloakProvider) {
p.realm = realm
}
}
// KeyCloak initializes a new keycloak provider
func KeyCloak(opts ...Option) (Provider, error) {
p := &KeycloakProvider{}
for _, opt := range opts {
opt(p)
}
if p.adminURL == "" {
return nil, errors.New("Admin URL cannot be empty")
}
_, err := url.Parse(p.adminURL)
if err != nil {
return nil, fmt.Errorf("Unable to parse the adminURL %s: %w", p.adminURL, err)
}
if p.client.Transport == nil {
p.client.Transport = http.DefaultTransport
}
if p.oeConfig.TokenEndpoint == "" {
return nil, errors.New("missing OpenID token endpoint")
}
if p.realm == "" {
p.realm = "master" // default realm
}
return p, nil
}

62
internal/config/identity/openid/provider/provider.go Normal file
View File

@@ -0,0 +1,62 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package provider
import "errors"
// DiscoveryDoc - parses the output from openid-configuration
// for example https://accounts.google.com/.well-known/openid-configuration
//
//nolint:unused
type DiscoveryDoc struct {
Issuer string `json:"issuer,omitempty"`
AuthEndpoint string `json:"authorization_endpoint,omitempty"`
TokenEndpoint string `json:"token_endpoint,omitempty"`
EndSessionEndpoint string `json:"end_session_endpoint,omitempty"`
UserInfoEndpoint string `json:"userinfo_endpoint,omitempty"`
RevocationEndpoint string `json:"revocation_endpoint,omitempty"`
JwksURI string `json:"jwks_uri,omitempty"`
ResponseTypesSupported []string `json:"response_types_supported,omitempty"`
SubjectTypesSupported []string `json:"subject_types_supported,omitempty"`
IDTokenSigningAlgValuesSupported []string `json:"id_token_signing_alg_values_supported,omitempty"`
ScopesSupported []string `json:"scopes_supported,omitempty"`
TokenEndpointAuthMethods []string `json:"token_endpoint_auth_methods_supported,omitempty"`
ClaimsSupported []string `json:"claims_supported,omitempty"`
CodeChallengeMethodsSupported []string `json:"code_challenge_methods_supported,omitempty"`
}
// User represents information about user.
type User struct {
Name string `json:"username"`
ID string `json:"id"`
Enabled bool `json:"enabled"`
}
// Standard errors.
var (
ErrNotImplemented = errors.New("function not implemented")
ErrAccessTokenExpired = errors.New("access_token expired or unauthorized")
)
// Provider implements identity provider specific admin operations, such as
// looking up users, fetching additional attributes etc.
type Provider interface {
LoginWithUser(username, password string) error
LoginWithClientID(clientID, clientSecret string) error
LookupUser(userid string) (User, error)
}

161
internal/config/identity/openid/providercfg.go Normal file
View File

@@ -0,0 +1,161 @@
// Copyright (c) 2015-2022 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package openid
import (
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"github.com/minio/minio/internal/arn"
"github.com/minio/minio/internal/config"
"github.com/minio/minio/internal/config/identity/openid/provider"
xhttp "github.com/minio/minio/internal/http"
xnet "github.com/minio/pkg/v3/net"
)
type providerCfg struct {
// Used for user interface like console
DisplayName string
JWKS struct {
URL *xnet.URL
}
URL *xnet.URL
ClaimPrefix string
ClaimName string
ClaimUserinfo bool
RedirectURI string
RedirectURIDynamic bool
DiscoveryDoc DiscoveryDoc
ClientID string
ClientSecret string
RolePolicy string
UserReadableClaim string
UserIDClaim string
roleArn arn.ARN
provider provider.Provider
}
func newProviderCfgFromConfig(getCfgVal func(cfgName string) string) providerCfg {
return providerCfg{
DisplayName: getCfgVal(DisplayName),
ClaimName: getCfgVal(ClaimName),
ClaimUserinfo: getCfgVal(ClaimUserinfo) == config.EnableOn,
ClaimPrefix: getCfgVal(ClaimPrefix),
RedirectURI: getCfgVal(RedirectURI),
RedirectURIDynamic: getCfgVal(RedirectURIDynamic) == config.EnableOn,
ClientID: getCfgVal(ClientID),
ClientSecret: getCfgVal(ClientSecret),
RolePolicy: getCfgVal(RolePolicy),
UserReadableClaim: getCfgVal(UserReadableClaim),
UserIDClaim: getCfgVal(UserIDClaim),
}
}
const (
keyCloakVendor = "keycloak"
)
// initializeProvider initializes if any additional vendor specific information
// was provided, initialization will return an error initial login fails.
func (p *providerCfg) initializeProvider(cfgGet func(string) string, transport http.RoundTripper) error {
vendor := cfgGet(Vendor)
if vendor == "" {
return nil
}
var err error
switch vendor {
case keyCloakVendor:
adminURL := cfgGet(KeyCloakAdminURL)
realm := cfgGet(KeyCloakRealm)
p.provider, err = provider.KeyCloak(
provider.WithAdminURL(adminURL),
provider.WithOpenIDConfig(provider.DiscoveryDoc(p.DiscoveryDoc)),
provider.WithTransport(transport),
provider.WithRealm(realm),
)
return err
default:
return fmt.Errorf("Unsupported vendor %s", keyCloakVendor)
}
}
// GetRoleArn returns the role ARN.
func (p *providerCfg) GetRoleArn() string {
if p.RolePolicy == "" {
return ""
}
return p.roleArn.String()
}
// UserInfo returns claims for authenticated user from userInfo endpoint.
//
// Some OIDC implementations such as GitLab do not support
// claims as part of the normal oauth2 flow, instead rely
// on service providers making calls to IDP to fetch additional
// claims available from the UserInfo endpoint
func (p *providerCfg) UserInfo(ctx context.Context, accessToken string, transport http.RoundTripper) (map[string]any, error) {
if p.JWKS.URL == nil || p.JWKS.URL.String() == "" {
return nil, errors.New("openid not configured")
}
req, err := http.NewRequestWithContext(ctx, http.MethodPost, p.DiscoveryDoc.UserInfoEndpoint, nil)
if err != nil {
return nil, err
}
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
if accessToken != "" {
req.Header.Set("Authorization", "Bearer "+accessToken)
}
client := &http.Client{
Transport: transport,
}
resp, err := client.Do(req)
if err != nil {
return nil, err
}
defer xhttp.DrainBody(resp.Body)
if resp.StatusCode != http.StatusOK {
// uncomment this for debugging when needed.
// reqBytes, _ := httputil.DumpRequest(req, false)
// fmt.Println(string(reqBytes))
// respBytes, _ := httputil.DumpResponse(resp, true)
// fmt.Println(string(respBytes))
return nil, errors.New(resp.Status)
}
claims := map[string]any{}
if err = json.NewDecoder(resp.Body).Decode(&claims); err != nil {
// uncomment this for debugging when needed.
// reqBytes, _ := httputil.DumpRequest(req, false)
// fmt.Println(string(reqBytes))
// respBytes, _ := httputil.DumpResponse(resp, true)
// fmt.Println(string(respBytes))
return nil, err
}
return claims, nil
}

51
internal/config/identity/openid/rsa-sha3_contrib.go Normal file
View File

@@ -0,0 +1,51 @@
// MinIO Object Storage (c) 2021 MinIO, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package openid
import (
"crypto"
"github.com/golang-jwt/jwt/v4"
// Needed for SHA3 to work - See: https://golang.org/src/crypto/crypto.go?s=1034:1288
_ "golang.org/x/crypto/sha3"
)
// Specific instances for RS256 and company
var (
SigningMethodRS3256 *jwt.SigningMethodRSA
SigningMethodRS3384 *jwt.SigningMethodRSA
SigningMethodRS3512 *jwt.SigningMethodRSA
)
func init() {
// RS3256
SigningMethodRS3256 = &jwt.SigningMethodRSA{Name: "RS3256", Hash: crypto.SHA3_256}
jwt.RegisterSigningMethod(SigningMethodRS3256.Alg(), func() jwt.SigningMethod {
return SigningMethodRS3256
})
// RS3384
SigningMethodRS3384 = &jwt.SigningMethodRSA{Name: "RS3384", Hash: crypto.SHA3_384}
jwt.RegisterSigningMethod(SigningMethodRS3384.Alg(), func() jwt.SigningMethod {
return SigningMethodRS3384
})
// RS3512
SigningMethodRS3512 = &jwt.SigningMethodRSA{Name: "RS3512", Hash: crypto.SHA3_512}
jwt.RegisterSigningMethod(SigningMethodRS3512.Alg(), func() jwt.SigningMethod {
return SigningMethodRS3512
})
}

514
internal/config/identity/plugin/config.go Normal file
View File

@@ -0,0 +1,514 @@
// Copyright (c) 2015-2022 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package plugin
import (
"bytes"
"context"
"crypto/sha1"
"encoding/base64"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"regexp"
"sync"
"time"
"github.com/minio/minio/internal/arn"
"github.com/minio/minio/internal/config"
"github.com/minio/minio/internal/logger"
"github.com/minio/pkg/v3/env"
xnet "github.com/minio/pkg/v3/net"
)
func authNLogIf(ctx context.Context, err error) {
logger.LogIf(ctx, "authN", err)
}
// Authentication Plugin config and env variables
const (
URL = "url"
AuthToken = "auth_token"
RolePolicy = "role_policy"
RoleID = "role_id"
EnvIdentityPluginURL = "MINIO_IDENTITY_PLUGIN_URL"
EnvIdentityPluginAuthToken = "MINIO_IDENTITY_PLUGIN_AUTH_TOKEN"
EnvIdentityPluginRolePolicy = "MINIO_IDENTITY_PLUGIN_ROLE_POLICY"
EnvIdentityPluginRoleID = "MINIO_IDENTITY_PLUGIN_ROLE_ID"
)
var (
// DefaultKVS - default config for AuthN plugin config
DefaultKVS = config.KVS{
config.KV{
Key: URL,
Value: "",
},
config.KV{
Key: AuthToken,
Value: "",
},
config.KV{
Key: RolePolicy,
Value: "",
},
config.KV{
Key: RoleID,
Value: "",
},
}
defaultHelpPostfix = func(key string) string {
return config.DefaultHelpPostfix(DefaultKVS, key)
}
// Help for Identity Plugin
Help = config.HelpKVS{
config.HelpKV{
Key: URL,
Description: `plugin hook endpoint (HTTP(S)) e.g. "http://localhost:8181/path/to/endpoint"` + defaultHelpPostfix(URL),
Type: "url",
},
config.HelpKV{
Key: AuthToken,
Description: "authorization token for plugin hook endpoint" + defaultHelpPostfix(AuthToken),
Optional: true,
Type: "string",
Sensitive: true,
Secret: true,
},
config.HelpKV{
Key: RolePolicy,
Description: "policies to apply for plugin authorized users" + defaultHelpPostfix(RolePolicy),
Type: "string",
},
config.HelpKV{
Key: RoleID,
Description: "unique ID to generate the ARN" + defaultHelpPostfix(RoleID),
Optional: true,
Type: "string",
},
config.HelpKV{
Key: config.Comment,
Description: config.DefaultComment,
Optional: true,
Type: "sentence",
},
}
)
// Allows only Base64 URL encoding characters.
var validRoleIDRegex = regexp.MustCompile(`^[A-Za-z0-9_-]+$`)
// Args for authentication plugin.
type Args struct {
URL *xnet.URL
AuthToken string
Transport http.RoundTripper
CloseRespFn func(r io.ReadCloser)
RolePolicy string
RoleARN arn.ARN
}
// Validate - validate configuration params.
func (a *Args) Validate() error {
req, err := http.NewRequest(http.MethodPost, a.URL.String(), bytes.NewReader([]byte("")))
if err != nil {
return err
}
req.Header.Set("Content-Type", "application/json")
if a.AuthToken != "" {
req.Header.Set("Authorization", a.AuthToken)
}
client := &http.Client{Transport: a.Transport}
resp, err := client.Do(req)
if err != nil {
return err
}
defer a.CloseRespFn(resp.Body)
return nil
}
type serviceRTTMinuteStats struct {
statsTime time.Time
rttMsSum, maxRttMs float64
successRequestCount int64
failedRequestCount int64
}
type metrics struct {
sync.Mutex
LastCheckSuccess time.Time
LastCheckFailure time.Time
lastFullMinute serviceRTTMinuteStats
currentMinute serviceRTTMinuteStats
}
func (h *metrics) setConnSuccess(reqStartTime time.Time) {
h.Lock()
defer h.Unlock()
h.LastCheckSuccess = reqStartTime
}
func (h *metrics) setConnFailure(reqStartTime time.Time) {
h.Lock()
defer h.Unlock()
h.LastCheckFailure = reqStartTime
}
func (h *metrics) updateLastFullMinute(currReqMinute time.Time) {
// Assumes the caller has h.Lock()'ed
h.lastFullMinute = h.currentMinute
h.currentMinute = serviceRTTMinuteStats{
statsTime: currReqMinute,
}
}
func (h *metrics) accumRequestRTT(reqStartTime time.Time, rttMs float64, isSuccess bool) {
h.Lock()
defer h.Unlock()
// Update connectivity times
if isSuccess {
if reqStartTime.After(h.LastCheckSuccess) {
h.LastCheckSuccess = reqStartTime
}
} else {
if reqStartTime.After(h.LastCheckFailure) {
h.LastCheckFailure = reqStartTime
}
}
// Round the request time *down* to whole minute.
reqTimeMinute := reqStartTime.Truncate(time.Minute)
if reqTimeMinute.After(h.currentMinute.statsTime) {
// Drop the last full minute now, since we got a request for a time we
// are not yet tracking.
h.updateLastFullMinute(reqTimeMinute)
}
var entry *serviceRTTMinuteStats
switch {
case reqTimeMinute.Equal(h.currentMinute.statsTime):
entry = &h.currentMinute
case reqTimeMinute.Equal(h.lastFullMinute.statsTime):
entry = &h.lastFullMinute
default:
// This request is too old, it should never happen, ignore it as we
// cannot return an error.
return
}
// Update stats
if isSuccess {
if entry.maxRttMs < rttMs {
entry.maxRttMs = rttMs
}
entry.rttMsSum += rttMs
entry.successRequestCount++
} else {
entry.failedRequestCount++
}
}
// AuthNPlugin - implements pluggable authentication via webhook.
type AuthNPlugin struct {
args Args
client *http.Client
shutdownCtx context.Context
serviceMetrics *metrics
}
// Enabled returns if AuthNPlugin is enabled.
func Enabled(kvs config.KVS) bool {
return kvs.Get(URL) != ""
}
// LookupConfig lookup AuthNPlugin from config, override with any ENVs.
func LookupConfig(kv config.KVS, transport *http.Transport, closeRespFn func(io.ReadCloser), serverRegion string) (Args, error) {
args := Args{}
if err := config.CheckValidKeys(config.IdentityPluginSubSys, kv, DefaultKVS); err != nil {
return args, err
}
pluginURL := env.Get(EnvIdentityPluginURL, kv.Get(URL))
if pluginURL == "" {
return args, nil
}
authToken := env.Get(EnvIdentityPluginAuthToken, kv.Get(AuthToken))
u, err := xnet.ParseHTTPURL(pluginURL)
if err != nil {
return args, err
}
rolePolicy := env.Get(EnvIdentityPluginRolePolicy, kv.Get(RolePolicy))
if rolePolicy == "" {
return args, config.Errorf("A role policy must be specified for Identity Management Plugin")
}
resourceID := "idmp-"
roleID := env.Get(EnvIdentityPluginRoleID, kv.Get(RoleID))
if roleID == "" {
// We use a hash of the plugin URL so that the ARN remains
// constant across restarts.
h := sha1.New()
h.Write([]byte(pluginURL))
bs := h.Sum(nil)
resourceID += base64.RawURLEncoding.EncodeToString(bs)
} else {
// Check that the roleID is restricted to URL safe characters
// (base64 URL encoding chars).
if !validRoleIDRegex.MatchString(roleID) {
return args, config.Errorf("Role ID must match the regexp `^[a-zA-Z0-9_-]+$`")
}
// Use the user provided ID here.
resourceID += roleID
}
roleArn, err := arn.NewIAMRoleARN(resourceID, serverRegion)
if err != nil {
return args, config.Errorf("unable to generate ARN from the plugin config: %v", err)
}
args = Args{
URL: u,
AuthToken: authToken,
Transport: transport,
CloseRespFn: closeRespFn,
RolePolicy: rolePolicy,
RoleARN: roleArn,
}
if err = args.Validate(); err != nil {
return args, err
}
return args, nil
}
// New - initializes Authorization Management Plugin.
func New(shutdownCtx context.Context, args Args) *AuthNPlugin {
if args.URL == nil || args.URL.Scheme == "" && args.AuthToken == "" {
return nil
}
plugin := AuthNPlugin{
args: args,
client: &http.Client{Transport: args.Transport},
shutdownCtx: shutdownCtx,
serviceMetrics: &metrics{
Mutex: sync.Mutex{},
LastCheckSuccess: time.Unix(0, 0),
LastCheckFailure: time.Unix(0, 0),
lastFullMinute: serviceRTTMinuteStats{},
currentMinute: serviceRTTMinuteStats{},
},
}
go plugin.doPeriodicHealthCheck()
return &plugin
}
// AuthNSuccessResponse - represents the response from the authentication plugin
// service.
type AuthNSuccessResponse struct {
User string `json:"user"`
MaxValiditySeconds int `json:"maxValiditySeconds"`
Claims map[string]any `json:"claims"`
}
// AuthNErrorResponse - represents an error response from the authN plugin.
type AuthNErrorResponse struct {
Reason string `json:"reason"`
}
// AuthNResponse - represents a result of the authentication operation.
type AuthNResponse struct {
Success *AuthNSuccessResponse
Failure *AuthNErrorResponse
}
const (
minValidityDurationSeconds int = 900
maxValidityDurationSeconds int = 365 * 24 * 3600
)
// Authenticate authenticates the token with the external hook endpoint and
// returns a parent user, max expiry duration for the authentication and a set
// of claims.
func (o *AuthNPlugin) Authenticate(roleArn arn.ARN, token string) (AuthNResponse, error) {
if o == nil {
return AuthNResponse{}, nil
}
if roleArn != o.args.RoleARN {
return AuthNResponse{}, fmt.Errorf("Invalid role ARN value: %s", roleArn.String())
}
u := url.URL(*o.args.URL)
q := u.Query()
q.Set("token", token)
u.RawQuery = q.Encode()
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
req, err := http.NewRequestWithContext(ctx, http.MethodPost, u.String(), nil)
if err != nil {
return AuthNResponse{}, err
}
if o.args.AuthToken != "" {
req.Header.Set("Authorization", o.args.AuthToken)
}
reqStartTime := time.Now()
resp, err := o.client.Do(req)
if err != nil {
o.serviceMetrics.accumRequestRTT(reqStartTime, 0, false)
return AuthNResponse{}, err
}
defer o.args.CloseRespFn(resp.Body)
reqDurNanos := time.Since(reqStartTime).Nanoseconds()
o.serviceMetrics.accumRequestRTT(reqStartTime, float64(reqDurNanos)/1e6, true)
switch resp.StatusCode {
case 200:
var result AuthNSuccessResponse
if err = json.NewDecoder(resp.Body).Decode(&result); err != nil {
return AuthNResponse{}, err
}
if result.MaxValiditySeconds < minValidityDurationSeconds || result.MaxValiditySeconds > maxValidityDurationSeconds {
return AuthNResponse{}, fmt.Errorf("Plugin returned an invalid validity duration (%d) - should be between %d and %d",
result.MaxValiditySeconds, minValidityDurationSeconds, maxValidityDurationSeconds)
}
return AuthNResponse{
Success: &result,
}, nil
case 403:
var result AuthNErrorResponse
if err = json.NewDecoder(resp.Body).Decode(&result); err != nil {
return AuthNResponse{}, err
}
return AuthNResponse{
Failure: &result,
}, nil
default:
return AuthNResponse{}, fmt.Errorf("Invalid status code %d from auth plugin", resp.StatusCode)
}
}
// GetRoleInfo - returns ARN to policies map.
func (o *AuthNPlugin) GetRoleInfo() map[arn.ARN]string {
return map[arn.ARN]string{
o.args.RoleARN: o.args.RolePolicy,
}
}
// checkConnectivity returns true if we are able to connect to the plugin
// service.
func (o *AuthNPlugin) checkConnectivity(ctx context.Context) bool {
ctx, cancel := context.WithTimeout(ctx, healthCheckTimeout)
defer cancel()
u := url.URL(*o.args.URL)
req, err := http.NewRequestWithContext(ctx, http.MethodHead, u.String(), nil)
if err != nil {
authNLogIf(ctx, err)
return false
}
if o.args.AuthToken != "" {
req.Header.Set("Authorization", o.args.AuthToken)
}
resp, err := o.client.Do(req)
if err != nil {
return false
}
defer o.args.CloseRespFn(resp.Body)
return true
}
var (
healthCheckInterval = 1 * time.Minute
healthCheckTimeout = 5 * time.Second
)
func (o *AuthNPlugin) doPeriodicHealthCheck() {
ticker := time.NewTicker(healthCheckInterval)
defer ticker.Stop()
for {
select {
case <-ticker.C:
now := time.Now()
isConnected := o.checkConnectivity(o.shutdownCtx)
if isConnected {
o.serviceMetrics.setConnSuccess(now)
} else {
o.serviceMetrics.setConnFailure(now)
}
case <-o.shutdownCtx.Done():
return
}
}
}
// Metrics contains metrics about the authentication plugin service.
type Metrics struct {
LastReachableSecs, LastUnreachableSecs float64
// Last whole minute stats
TotalRequests, FailedRequests int64
AvgSuccRTTMs float64
MaxSuccRTTMs float64
}
// Metrics reports metrics related to plugin service reachability and stats for the last whole minute
func (o *AuthNPlugin) Metrics() Metrics {
if o == nil {
// Return empty metrics when not configured.
return Metrics{}
}
o.serviceMetrics.Lock()
defer o.serviceMetrics.Unlock()
l := &o.serviceMetrics.lastFullMinute
var avg float64
if l.successRequestCount > 0 {
avg = l.rttMsSum / float64(l.successRequestCount)
}
now := time.Now().UTC()
return Metrics{
LastReachableSecs: now.Sub(o.serviceMetrics.LastCheckSuccess).Seconds(),
LastUnreachableSecs: now.Sub(o.serviceMetrics.LastCheckFailure).Seconds(),
TotalRequests: l.failedRequestCount + l.successRequestCount,
FailedRequests: l.failedRequestCount,
AvgSuccRTTMs: avg,
MaxSuccRTTMs: l.maxRttMs,
}
}

126
internal/config/identity/tls/config.go Normal file
View File

@@ -0,0 +1,126 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package tls
import (
"strconv"
"time"
"github.com/minio/minio/internal/auth"
"github.com/minio/minio/internal/config"
"github.com/minio/pkg/v3/env"
)
const (
// EnvIdentityTLSEnabled is an environment variable that controls whether the X.509
// TLS STS API is enabled. By default, if not set, it is enabled.
EnvIdentityTLSEnabled = "MINIO_IDENTITY_TLS_ENABLE"
// EnvIdentityTLSSkipVerify is an environment variable that controls whether
// MinIO verifies the client certificate present by the client
// when requesting temp. credentials.
// By default, MinIO always verify the client certificate.
//
// The client certificate verification should only be skipped
// when debugging or testing a setup since it allows arbitrary
// clients to obtain temp. credentials with arbitrary policy
// permissions - including admin permissions.
EnvIdentityTLSSkipVerify = "MINIO_IDENTITY_TLS_SKIP_VERIFY"
)
// Config contains the STS TLS configuration for generating temp.
// credentials and mapping client certificates to S3 policies.
type Config struct {
Enabled bool `json:"enabled"`
// InsecureSkipVerify, if set to true, disables the client
// certificate verification. It should only be set for
// debugging or testing purposes.
InsecureSkipVerify bool `json:"skip_verify"`
}
const (
defaultExpiry time.Duration = 1 * time.Hour
minExpiry time.Duration = 15 * time.Minute
maxExpiry time.Duration = 365 * 24 * time.Hour
)
// GetExpiryDuration - return parsed expiry duration.
func (l Config) GetExpiryDuration(dsecs string) (time.Duration, error) {
if dsecs == "" {
return defaultExpiry, nil
}
d, err := strconv.Atoi(dsecs)
if err != nil {
return 0, auth.ErrInvalidDuration
}
dur := time.Duration(d) * time.Second
if dur < minExpiry || dur > maxExpiry {
return 0, auth.ErrInvalidDuration
}
return dur, nil
}
// Lookup returns a new Config by merging the given K/V config
// system with environment variables.
func Lookup(kvs config.KVS) (Config, error) {
if err := config.CheckValidKeys(config.IdentityTLSSubSys, kvs, DefaultKVS); err != nil {
return Config{}, err
}
cfg := Config{}
var err error
v := env.Get(EnvIdentityTLSEnabled, "")
if v == "" {
return cfg, nil
}
cfg.Enabled, err = config.ParseBool(v)
if err != nil {
return Config{}, err
}
cfg.InsecureSkipVerify, err = config.ParseBool(env.Get(EnvIdentityTLSSkipVerify, kvs.Get(skipVerify)))
if err != nil {
return Config{}, err
}
return cfg, nil
}
const (
skipVerify = "skip_verify"
)
// DefaultKVS is the default K/V config system for
// the STS TLS API.
var DefaultKVS = config.KVS{
config.KV{
Key: skipVerify,
Value: "off",
},
}
// Help is the help and description for the STS API K/V configuration.
var Help = config.HelpKVS{
config.HelpKV{
Key: skipVerify,
Description: `trust client certificates without verification (default: 'off')`,
Optional: true,
Type: "on|off",
},
}

52
internal/config/ilm/help.go Normal file
View File

@@ -0,0 +1,52 @@
// Copyright (c) 2015-2024 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package ilm
import "github.com/minio/minio/internal/config"
const (
transitionWorkers = "transition_workers"
expirationWorkers = "expiration_workers"
// EnvILMTransitionWorkers env variable to configure number of transition workers
EnvILMTransitionWorkers = "MINIO_ILM_TRANSITION_WORKERS"
// EnvILMExpirationWorkers env variable to configure number of expiration workers
EnvILMExpirationWorkers = "MINIO_ILM_EXPIRATION_WORKERS"
)
var (
defaultHelpPostfix = func(key string) string {
return config.DefaultHelpPostfix(DefaultKVS, key)
}
// Help holds configuration keys and their default values for the ILM
// subsystem
Help = config.HelpKVS{
config.HelpKV{
Key: transitionWorkers,
Type: "number",
Description: `set the number of transition workers` + defaultHelpPostfix(transitionWorkers),
Optional: true,
},
config.HelpKV{
Key: expirationWorkers,
Type: "number",
Description: `set the number of expiration workers` + defaultHelpPostfix(expirationWorkers),
Optional: true,
},
}
)

69
internal/config/ilm/ilm.go Normal file
View File

@@ -0,0 +1,69 @@
// Copyright (c) 2015-2024 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package ilm
import (
"strconv"
"github.com/minio/minio/internal/config"
"github.com/minio/pkg/v3/env"
)
// DefaultKVS default configuration values for ILM subsystem
var DefaultKVS = config.KVS{
config.KV{
Key: transitionWorkers,
Value: "100",
},
config.KV{
Key: expirationWorkers,
Value: "100",
},
}
// Config represents the different configuration values for ILM subsystem
type Config struct {
TransitionWorkers int
ExpirationWorkers int
}
// LookupConfig - lookup ilm config and override with valid environment settings if any.
func LookupConfig(kvs config.KVS) (cfg Config, err error) {
cfg = Config{
TransitionWorkers: 100,
ExpirationWorkers: 100,
}
if err = config.CheckValidKeys(config.ILMSubSys, kvs, DefaultKVS); err != nil {
return cfg, err
}
tw, err := strconv.Atoi(env.Get(EnvILMTransitionWorkers, kvs.GetWithDefault(transitionWorkers, DefaultKVS)))
if err != nil {
return cfg, err
}
ew, err := strconv.Atoi(env.Get(EnvILMExpirationWorkers, kvs.GetWithDefault(expirationWorkers, DefaultKVS)))
if err != nil {
return cfg, err
}
cfg.TransitionWorkers = tw
cfg.ExpirationWorkers = ew
return cfg, nil
}

40
internal/config/lambda/config.go Normal file
View File

@@ -0,0 +1,40 @@
// Copyright (c) 2015-2023 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package lambda
import "github.com/minio/minio/internal/event/target"
// Config - lambda target configuration structure, holds
// information about various lambda targets.
type Config struct {
Webhook map[string]target.WebhookArgs `json:"webhook"`
}
const (
defaultTarget = "1"
)
// NewConfig - initialize lambda config.
func NewConfig() Config {
// Make sure to initialize lambda targets
cfg := Config{
Webhook: make(map[string]target.WebhookArgs),
}
cfg.Webhook[defaultTarget] = target.WebhookArgs{}
return cfg
}

62
internal/config/lambda/event/arn.go Normal file
View File

@@ -0,0 +1,62 @@
// Copyright (c) 2015-2023 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package event
import (
"strings"
)
// ARN - SQS resource name representation.
type ARN struct {
TargetID
region string
}
// String - returns string representation.
func (arn ARN) String() string {
if arn.ID == "" && arn.Name == "" && arn.region == "" {
return ""
}
return "arn:minio:s3-object-lambda:" + arn.region + ":" + arn.TargetID.String()
}
// ParseARN - parses string to ARN.
func ParseARN(s string) (*ARN, error) {
// ARN must be in the format of arn:minio:s3-object-lambda:<REGION>:<ID>:<TYPE>
if !strings.HasPrefix(s, "arn:minio:s3-object-lambda:") {
return nil, &ErrInvalidARN{s}
}
tokens := strings.Split(s, ":")
if len(tokens) != 6 {
return nil, &ErrInvalidARN{s}
}
if tokens[4] == "" || tokens[5] == "" {
return nil, &ErrInvalidARN{s}
}
return &ARN{
region: tokens[3],
TargetID: TargetID{
ID: tokens[4],
Name: tokens[5],
},
}, nil
}

72
internal/config/lambda/event/arn_test.go Normal file
View File

@@ -0,0 +1,72 @@
// Copyright (c) 2015-2023 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package event
import (
"testing"
)
func TestARNString(t *testing.T) {
testCases := []struct {
arn ARN
expectedResult string
}{
{ARN{}, ""},
{ARN{TargetID{"1", "webhook"}, ""}, "arn:minio:s3-object-lambda::1:webhook"},
{ARN{TargetID{"1", "webhook"}, "us-east-1"}, "arn:minio:s3-object-lambda:us-east-1:1:webhook"},
}
for i, testCase := range testCases {
result := testCase.arn.String()
if result != testCase.expectedResult {
t.Fatalf("test %v: result: expected: %v, got: %v", i+1, testCase.expectedResult, result)
}
}
}
func TestParseARN(t *testing.T) {
testCases := []struct {
s string
expectedARN *ARN
expectErr bool
}{
{"", nil, true},
{"arn:minio:s3-object-lambda:::", nil, true},
{"arn:minio:s3-object-lambda::1:webhook:remote", nil, true},
{"arn:aws:s3-object-lambda::1:webhook", nil, true},
{"arn:minio:sns::1:webhook", nil, true},
{"arn:minio:s3-object-lambda::1:webhook", &ARN{TargetID{"1", "webhook"}, ""}, false},
{"arn:minio:s3-object-lambda:us-east-1:1:webhook", &ARN{TargetID{"1", "webhook"}, "us-east-1"}, false},
}
for i, testCase := range testCases {
arn, err := ParseARN(testCase.s)
expectErr := (err != nil)
if expectErr != testCase.expectErr {
t.Fatalf("test %v: error: expected: %v, got: %v", i+1, testCase.expectErr, expectErr)
}
if !testCase.expectErr {
if *arn != *testCase.expectedARN {
t.Fatalf("test %v: data: expected: %v, got: %v", i+1, testCase.expectedARN, arn)
}
}
}
}

49
internal/config/lambda/event/errors.go Normal file
View File

@@ -0,0 +1,49 @@
// Copyright (c) 2015-2023 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package event
import (
"fmt"
)
// ErrUnknownRegion - unknown region error.
type ErrUnknownRegion struct {
Region string
}
func (err ErrUnknownRegion) Error() string {
return fmt.Sprintf("unknown region '%v'", err.Region)
}
// ErrARNNotFound - ARN not found error.
type ErrARNNotFound struct {
ARN ARN
}
func (err ErrARNNotFound) Error() string {
return fmt.Sprintf("ARN '%v' not found", err.ARN)
}
// ErrInvalidARN - invalid ARN error.
type ErrInvalidARN struct {
ARN string
}
func (err ErrInvalidARN) Error() string {
return fmt.Sprintf("invalid ARN '%v'", err.ARN)
}

80
internal/config/lambda/event/event.go Normal file
View File

@@ -0,0 +1,80 @@
// Copyright (c) 2015-2023 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package event
import "net/http"
// Identity represents access key who caused the event.
type Identity struct {
Type string `json:"type"`
PrincipalID string `json:"principalId"`
AccessKeyID string `json:"accessKeyId"`
}
// UserRequest user request headers
type UserRequest struct {
URL string `json:"url"`
Headers http.Header `json:"headers"`
}
// GetObjectContext provides the necessary details to perform
// download of the object, and return back the processed response
// to the server.
type GetObjectContext struct {
OutputRoute string `json:"outputRoute"`
OutputToken string `json:"outputToken"`
InputS3URL string `json:"inputS3Url"`
}
// Event represents lambda function event, this is undocumented in AWS S3. This
// structure bases itself on this structure but there is no binding.
//
// {
// "xAmzRequestId": "a2871150-1df5-4dc9-ad9f-3da283ca1bf3",
// "getObjectContext": {
// "outputRoute": "...",
// "outputToken": "...",
// "inputS3Url": "<presignedURL>"
// },
// "configuration": { // not useful in MinIO
// "accessPointArn": "...",
// "supportingAccessPointArn": "...",
// "payload": ""
// },
// "userRequest": {
// "url": "...",
// "headers": {
// "Host": "...",
// "X-Amz-Content-SHA256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
// }
// },
// "userIdentity": {
// "type": "IAMUser",
// "principalId": "AIDAJF5MO57RFXQCE5ZNC",
// "arn": "...",
// "accountId": "...",
// "accessKeyId": "AKIA3WNQJCXE2DYPAU7R"
// },
// "protocolVersion": "1.00"
// }
type Event struct {
ProtocolVersion string `json:"protocolVersion"`
GetObjectContext *GetObjectContext `json:"getObjectContext"`
UserIdentity Identity `json:"userIdentity"`
UserRequest UserRequest `json:"userRequest"`
}

74
internal/config/lambda/event/targetid.go Normal file
View File

@@ -0,0 +1,74 @@
// Copyright (c) 2015-2023 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package event
import (
"encoding/json"
"fmt"
"strings"
)
// TargetID - holds identification and name strings of notification target.
type TargetID struct {
ID string
Name string
}
// String - returns string representation.
func (tid TargetID) String() string {
return tid.ID + ":" + tid.Name
}
// ToARN - converts to ARN.
func (tid TargetID) ToARN(region string) ARN {
return ARN{TargetID: tid, region: region}
}
// MarshalJSON - encodes to JSON data.
func (tid TargetID) MarshalJSON() ([]byte, error) {
return json.Marshal(tid.String())
}
// UnmarshalJSON - decodes JSON data.
func (tid *TargetID) UnmarshalJSON(data []byte) error {
var s string
if err := json.Unmarshal(data, &s); err != nil {
return err
}
targetID, err := parseTargetID(s)
if err != nil {
return err
}
*tid = *targetID
return nil
}
// parseTargetID - parses string to TargetID.
func parseTargetID(s string) (*TargetID, error) {
tokens := strings.Split(s, ":")
if len(tokens) != 2 {
return nil, fmt.Errorf("invalid TargetID format '%v'", s)
}
return &TargetID{
ID: tokens[0],
Name: tokens[1],
}, nil
}

118
internal/config/lambda/event/targetid_test.go Normal file
View File

@@ -0,0 +1,118 @@
// Copyright (c) 2015-2023 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package event
import (
"reflect"
"testing"
)
func TestTargetDString(t *testing.T) {
testCases := []struct {
tid TargetID
expectedResult string
}{
{TargetID{}, ":"},
{TargetID{"1", "webhook"}, "1:webhook"},
{TargetID{"httpclient+2e33cdee-fbec-4bdd-917e-7d8e3c5a2531", "localhost:55638"}, "httpclient+2e33cdee-fbec-4bdd-917e-7d8e3c5a2531:localhost:55638"},
}
for i, testCase := range testCases {
result := testCase.tid.String()
if result != testCase.expectedResult {
t.Fatalf("test %v: result: expected: %v, got: %v", i+1, testCase.expectedResult, result)
}
}
}
func TestTargetDToARN(t *testing.T) {
tid := TargetID{"1", "webhook"}
testCases := []struct {
tid TargetID
region string
expectedARN ARN
}{
{tid, "", ARN{TargetID: tid, region: ""}},
{tid, "us-east-1", ARN{TargetID: tid, region: "us-east-1"}},
}
for i, testCase := range testCases {
arn := testCase.tid.ToARN(testCase.region)
if arn != testCase.expectedARN {
t.Fatalf("test %v: ARN: expected: %v, got: %v", i+1, testCase.expectedARN, arn)
}
}
}
func TestTargetDMarshalJSON(t *testing.T) {
testCases := []struct {
tid TargetID
expectedData []byte
expectErr bool
}{
{TargetID{}, []byte(`":"`), false},
{TargetID{"1", "webhook"}, []byte(`"1:webhook"`), false},
{TargetID{"httpclient+2e33cdee-fbec-4bdd-917e-7d8e3c5a2531", "localhost:55638"}, []byte(`"httpclient+2e33cdee-fbec-4bdd-917e-7d8e3c5a2531:localhost:55638"`), false},
}
for i, testCase := range testCases {
data, err := testCase.tid.MarshalJSON()
expectErr := (err != nil)
if expectErr != testCase.expectErr {
t.Fatalf("test %v: error: expected: %v, got: %v", i+1, testCase.expectErr, expectErr)
}
if !testCase.expectErr {
if !reflect.DeepEqual(data, testCase.expectedData) {
t.Fatalf("test %v: data: expected: %v, got: %v", i+1, string(testCase.expectedData), string(data))
}
}
}
}
func TestTargetDUnmarshalJSON(t *testing.T) {
testCases := []struct {
data []byte
expectedTargetID *TargetID
expectErr bool
}{
{[]byte(`""`), nil, true},
{[]byte(`"httpclient+2e33cdee-fbec-4bdd-917e-7d8e3c5a2531:localhost:55638"`), nil, true},
{[]byte(`":"`), &TargetID{}, false},
{[]byte(`"1:webhook"`), &TargetID{"1", "webhook"}, false},
}
for i, testCase := range testCases {
targetID := &TargetID{}
err := targetID.UnmarshalJSON(testCase.data)
expectErr := (err != nil)
if expectErr != testCase.expectErr {
t.Fatalf("test %v: error: expected: %v, got: %v", i+1, testCase.expectErr, expectErr)
}
if !testCase.expectErr {
if *targetID != *testCase.expectedTargetID {
t.Fatalf("test %v: TargetID: expected: %v, got: %v", i+1, testCase.expectedTargetID, targetID)
}
}
}
}

72
internal/config/lambda/event/targetidset.go Normal file
View File

@@ -0,0 +1,72 @@
// Copyright (c) 2015-2023 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package event
import "maps"
// TargetIDSet - Set representation of TargetIDs.
type TargetIDSet map[TargetID]struct{}
// IsEmpty returns true if the set is empty.
func (set TargetIDSet) IsEmpty() bool {
return len(set) != 0
}
// Clone - returns copy of this set.
func (set TargetIDSet) Clone() TargetIDSet {
setCopy := NewTargetIDSet()
maps.Copy(setCopy, set)
return setCopy
}
// add - adds TargetID to the set.
func (set TargetIDSet) add(targetID TargetID) {
set[targetID] = struct{}{}
}
// Union - returns union with given set as new set.
func (set TargetIDSet) Union(sset TargetIDSet) TargetIDSet {
nset := set.Clone()
for k := range sset {
nset.add(k)
}
return nset
}
// Difference - returns difference with given set as new set.
func (set TargetIDSet) Difference(sset TargetIDSet) TargetIDSet {
nset := NewTargetIDSet()
for k := range set {
if _, ok := sset[k]; !ok {
nset.add(k)
}
}
return nset
}
// NewTargetIDSet - creates new TargetID set with given TargetIDs.
func NewTargetIDSet(targetIDs ...TargetID) TargetIDSet {
set := make(TargetIDSet)
for _, targetID := range targetIDs {
set.add(targetID)
}
return set
}

110
internal/config/lambda/event/targetidset_test.go Normal file
View File

@@ -0,0 +1,110 @@
// Copyright (c) 2015-2023 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package event
import (
"reflect"
"testing"
)
func TestTargetIDSetClone(t *testing.T) {
testCases := []struct {
set TargetIDSet
targetIDToAdd TargetID
}{
{NewTargetIDSet(), TargetID{"1", "webhook"}},
{NewTargetIDSet(TargetID{"1", "webhook"}), TargetID{"2", "webhook"}},
{NewTargetIDSet(TargetID{"1", "webhook"}, TargetID{"2", "amqp"}), TargetID{"2", "webhook"}},
}
for i, testCase := range testCases {
result := testCase.set.Clone()
if !reflect.DeepEqual(result, testCase.set) {
t.Fatalf("test %v: result: expected: %v, got: %v", i+1, testCase.set, result)
}
result.add(testCase.targetIDToAdd)
if reflect.DeepEqual(result, testCase.set) {
t.Fatalf("test %v: result: expected: not equal, got: equal", i+1)
}
}
}
func TestTargetIDSetUnion(t *testing.T) {
testCases := []struct {
set TargetIDSet
setToAdd TargetIDSet
expectedResult TargetIDSet
}{
{NewTargetIDSet(), NewTargetIDSet(), NewTargetIDSet()},
{NewTargetIDSet(), NewTargetIDSet(TargetID{"1", "webhook"}), NewTargetIDSet(TargetID{"1", "webhook"})},
{NewTargetIDSet(TargetID{"1", "webhook"}), NewTargetIDSet(), NewTargetIDSet(TargetID{"1", "webhook"})},
{NewTargetIDSet(TargetID{"1", "webhook"}), NewTargetIDSet(TargetID{"2", "amqp"}), NewTargetIDSet(TargetID{"1", "webhook"}, TargetID{"2", "amqp"})},
{NewTargetIDSet(TargetID{"1", "webhook"}), NewTargetIDSet(TargetID{"1", "webhook"}), NewTargetIDSet(TargetID{"1", "webhook"})},
}
for i, testCase := range testCases {
result := testCase.set.Union(testCase.setToAdd)
if !reflect.DeepEqual(testCase.expectedResult, result) {
t.Fatalf("test %v: result: expected: %v, got: %v", i+1, testCase.expectedResult, result)
}
}
}
func TestTargetIDSetDifference(t *testing.T) {
testCases := []struct {
set TargetIDSet
setToRemove TargetIDSet
expectedResult TargetIDSet
}{
{NewTargetIDSet(), NewTargetIDSet(), NewTargetIDSet()},
{NewTargetIDSet(), NewTargetIDSet(TargetID{"1", "webhook"}), NewTargetIDSet()},
{NewTargetIDSet(TargetID{"1", "webhook"}), NewTargetIDSet(), NewTargetIDSet(TargetID{"1", "webhook"})},
{NewTargetIDSet(TargetID{"1", "webhook"}), NewTargetIDSet(TargetID{"2", "amqp"}), NewTargetIDSet(TargetID{"1", "webhook"})},
{NewTargetIDSet(TargetID{"1", "webhook"}), NewTargetIDSet(TargetID{"1", "webhook"}), NewTargetIDSet()},
}
for i, testCase := range testCases {
result := testCase.set.Difference(testCase.setToRemove)
if !reflect.DeepEqual(testCase.expectedResult, result) {
t.Fatalf("test %v: result: expected: %v, got: %v", i+1, testCase.expectedResult, result)
}
}
}
func TestNewTargetIDSet(t *testing.T) {
testCases := []struct {
targetIDs []TargetID
expectedResult TargetIDSet
}{
{[]TargetID{}, NewTargetIDSet()},
{[]TargetID{{"1", "webhook"}}, NewTargetIDSet(TargetID{"1", "webhook"})},
{[]TargetID{{"1", "webhook"}, {"2", "amqp"}}, NewTargetIDSet(TargetID{"1", "webhook"}, TargetID{"2", "amqp"})},
}
for i, testCase := range testCases {
result := NewTargetIDSet(testCase.targetIDs...)
if !reflect.DeepEqual(testCase.expectedResult, result) {
t.Fatalf("test %v: result: expected: %v, got: %v", i+1, testCase.expectedResult, result)
}
}
}

188
internal/config/lambda/event/targetlist.go Normal file
View File

@@ -0,0 +1,188 @@
// Copyright (c) 2015-2023 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package event
import (
"fmt"
"maps"
"net/http"
"strings"
"sync"
)
// Target - lambda target interface
type Target interface {
ID() TargetID
IsActive() (bool, error)
Send(Event) (*http.Response, error)
Stat() TargetStat
Close() error
}
// TargetStats is a collection of stats for multiple targets.
type TargetStats struct {
TargetStats map[string]TargetStat
}
// TargetStat is the stats of a single target.
type TargetStat struct {
ID TargetID
ActiveRequests int64
TotalRequests int64
FailedRequests int64
}
// TargetList - holds list of targets indexed by target ID.
type TargetList struct {
sync.RWMutex
targets map[TargetID]Target
}
// Add - adds unique target to target list.
func (list *TargetList) Add(targets ...Target) error {
list.Lock()
defer list.Unlock()
for _, target := range targets {
if _, ok := list.targets[target.ID()]; ok {
return fmt.Errorf("target %v already exists", target.ID())
}
list.targets[target.ID()] = target
}
return nil
}
// Lookup - checks whether target by target ID exists is valid or not.
func (list *TargetList) Lookup(arnStr string) (Target, error) {
list.RLock()
defer list.RUnlock()
arn, err := ParseARN(arnStr)
if err != nil {
return nil, err
}
id, found := list.targets[arn.TargetID]
if !found {
return nil, &ErrARNNotFound{}
}
return id, nil
}
// TargetIDResult returns result of Remove/Send operation, sets err if
// any for the associated TargetID
type TargetIDResult struct {
// ID where the remove or send were initiated.
ID TargetID
// Stores any error while removing a target or while sending an event.
Err error
}
// Remove - closes and removes targets by given target IDs.
func (list *TargetList) Remove(targetIDSet TargetIDSet) {
list.Lock()
defer list.Unlock()
for id := range targetIDSet {
target, ok := list.targets[id]
if ok {
target.Close()
delete(list.targets, id)
}
}
}
// Targets - list all targets
func (list *TargetList) Targets() []Target {
if list == nil {
return []Target{}
}
list.RLock()
defer list.RUnlock()
targets := make([]Target, 0, len(list.targets))
for _, tgt := range list.targets {
targets = append(targets, tgt)
}
return targets
}
// Empty returns true if targetList is empty.
func (list *TargetList) Empty() bool {
list.RLock()
defer list.RUnlock()
return len(list.targets) == 0
}
// List - returns available target IDs.
func (list *TargetList) List(region string) []ARN {
list.RLock()
defer list.RUnlock()
keys := make([]ARN, 0, len(list.targets))
for k := range list.targets {
keys = append(keys, k.ToARN(region))
}
return keys
}
// TargetMap - returns available targets.
func (list *TargetList) TargetMap() map[TargetID]Target {
list.RLock()
defer list.RUnlock()
ntargets := make(map[TargetID]Target, len(list.targets))
maps.Copy(ntargets, list.targets)
return ntargets
}
// Send - sends events to targets identified by target IDs.
func (list *TargetList) Send(event Event, id TargetID) (*http.Response, error) {
list.RLock()
target, ok := list.targets[id]
list.RUnlock()
if ok {
return target.Send(event)
}
return nil, ErrARNNotFound{}
}
// Stats returns stats for targets.
func (list *TargetList) Stats() TargetStats {
t := TargetStats{}
if list == nil {
return t
}
list.RLock()
defer list.RUnlock()
t.TargetStats = make(map[string]TargetStat, len(list.targets))
for id, target := range list.targets {
t.TargetStats[strings.ReplaceAll(id.String(), ":", "_")] = target.Stat()
}
return t
}
// NewTargetList - creates TargetList.
func NewTargetList() *TargetList {
return &TargetList{targets: make(map[TargetID]Target)}
}

63
internal/config/lambda/help.go Normal file
View File

@@ -0,0 +1,63 @@
// Copyright (c) 2015-2023 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package lambda
import (
"github.com/minio/minio/internal/config"
"github.com/minio/minio/internal/event/target"
)
// Help template inputs for all lambda targets
var (
HelpWebhook = config.HelpKVS{
config.HelpKV{
Key: target.WebhookEndpoint,
Description: "webhook server endpoint e.g. http://localhost:8080/minio/lambda",
Type: "url",
Sensitive: true,
},
config.HelpKV{
Key: target.WebhookAuthToken,
Description: "opaque string or JWT authorization token",
Optional: true,
Type: "string",
Sensitive: true,
Secret: true,
},
config.HelpKV{
Key: config.Comment,
Description: config.DefaultComment,
Optional: true,
Type: "sentence",
},
config.HelpKV{
Key: target.WebhookClientCert,
Description: "client cert for Webhook mTLS auth",
Optional: true,
Type: "string",
Sensitive: true,
},
config.HelpKV{
Key: target.WebhookClientKey,
Description: "client cert key for Webhook mTLS auth",
Optional: true,
Type: "string",
Sensitive: true,
},
}
)

219
internal/config/lambda/parse.go Normal file
View File

@@ -0,0 +1,219 @@
// Copyright (c) 2015-2023 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package lambda
import (
"context"
"errors"
"fmt"
"net/http"
"github.com/minio/minio/internal/config"
"github.com/minio/minio/internal/config/lambda/event"
"github.com/minio/minio/internal/config/lambda/target"
"github.com/minio/minio/internal/logger"
"github.com/minio/pkg/v3/env"
xnet "github.com/minio/pkg/v3/net"
)
const (
logSubsys = "notify"
)
func logOnceIf(ctx context.Context, err error, id string, errKind ...any) {
logger.LogOnceIf(ctx, logSubsys, err, id, errKind...)
}
// ErrTargetsOffline - Indicates single/multiple target failures.
var ErrTargetsOffline = errors.New("one or more targets are offline. Please use `mc admin info --json` to check the offline targets")
// TestSubSysLambdaTargets - tests notification targets of given subsystem
func TestSubSysLambdaTargets(ctx context.Context, cfg config.Config, subSys string, transport *http.Transport) error {
if err := checkValidLambdaKeysForSubSys(subSys, cfg[subSys]); err != nil {
return err
}
targetList, err := fetchSubSysTargets(ctx, cfg, subSys, transport)
if err != nil {
return err
}
for _, target := range targetList {
defer target.Close()
}
for _, target := range targetList {
yes, err := target.IsActive()
if err == nil && !yes {
err = ErrTargetsOffline
}
if err != nil {
return fmt.Errorf("error (%s): %w", target.ID(), err)
}
}
return nil
}
func fetchSubSysTargets(ctx context.Context, cfg config.Config, subSys string, transport *http.Transport) (targets []event.Target, err error) {
if err := checkValidLambdaKeysForSubSys(subSys, cfg[subSys]); err != nil {
return nil, err
}
if subSys == config.LambdaWebhookSubSys {
webhookTargets, err := GetLambdaWebhook(cfg[config.LambdaWebhookSubSys], transport)
if err != nil {
return nil, err
}
for id, args := range webhookTargets {
if !args.Enable {
continue
}
t, err := target.NewWebhookTarget(ctx, id, args, logOnceIf, transport)
if err != nil {
return nil, err
}
targets = append(targets, t)
}
}
return targets, nil
}
// FetchEnabledTargets - Returns a set of configured TargetList
func FetchEnabledTargets(ctx context.Context, cfg config.Config, transport *http.Transport) (*event.TargetList, error) {
targetList := event.NewTargetList()
for _, subSys := range config.LambdaSubSystems.ToSlice() {
targets, err := fetchSubSysTargets(ctx, cfg, subSys, transport)
if err != nil {
return nil, err
}
for _, t := range targets {
if err = targetList.Add(t); err != nil {
return nil, err
}
}
}
return targetList, nil
}
// DefaultLambdaKVS - default notification list of kvs.
var (
DefaultLambdaKVS = map[string]config.KVS{
config.LambdaWebhookSubSys: DefaultWebhookKVS,
}
)
// DefaultWebhookKVS - default KV for webhook config
var (
DefaultWebhookKVS = config.KVS{
config.KV{
Key: config.Enable,
Value: config.EnableOff,
},
config.KV{
Key: target.WebhookEndpoint,
Value: "",
},
config.KV{
Key: target.WebhookAuthToken,
Value: "",
},
config.KV{
Key: target.WebhookClientCert,
Value: "",
},
config.KV{
Key: target.WebhookClientKey,
Value: "",
},
}
)
func checkValidLambdaKeysForSubSys(subSys string, tgt map[string]config.KVS) error {
validKVS, ok := DefaultLambdaKVS[subSys]
if !ok {
return nil
}
for tname, kv := range tgt {
subSysTarget := subSys
if tname != config.Default {
subSysTarget = subSys + config.SubSystemSeparator + tname
}
if v, ok := kv.Lookup(config.Enable); ok && v == config.EnableOn {
if err := config.CheckValidKeys(subSysTarget, kv, validKVS); err != nil {
return err
}
}
}
return nil
}
// GetLambdaWebhook - returns a map of registered notification 'webhook' targets
func GetLambdaWebhook(webhookKVS map[string]config.KVS, transport *http.Transport) (
map[string]target.WebhookArgs, error,
) {
webhookTargets := make(map[string]target.WebhookArgs)
for k, kv := range config.Merge(webhookKVS, target.EnvWebhookEnable, DefaultWebhookKVS) {
enableEnv := target.EnvWebhookEnable
if k != config.Default {
enableEnv = enableEnv + config.Default + k
}
enabled, err := config.ParseBool(env.Get(enableEnv, kv.Get(config.Enable)))
if err != nil {
return nil, err
}
if !enabled {
continue
}
urlEnv := target.EnvWebhookEndpoint
if k != config.Default {
urlEnv = urlEnv + config.Default + k
}
url, err := xnet.ParseHTTPURL(env.Get(urlEnv, kv.Get(target.WebhookEndpoint)))
if err != nil {
return nil, err
}
authEnv := target.EnvWebhookAuthToken
if k != config.Default {
authEnv = authEnv + config.Default + k
}
clientCertEnv := target.EnvWebhookClientCert
if k != config.Default {
clientCertEnv = clientCertEnv + config.Default + k
}
clientKeyEnv := target.EnvWebhookClientKey
if k != config.Default {
clientKeyEnv = clientKeyEnv + config.Default + k
}
webhookArgs := target.WebhookArgs{
Enable: enabled,
Endpoint: *url,
Transport: transport,
AuthToken: env.Get(authEnv, kv.Get(target.WebhookAuthToken)),
ClientCert: env.Get(clientCertEnv, kv.Get(target.WebhookClientCert)),
ClientKey: env.Get(clientKeyEnv, kv.Get(target.WebhookClientKey)),
}
if err = webhookArgs.Validate(); err != nil {
return nil, err
}
webhookTargets[k] = webhookArgs
}
return webhookTargets, nil
}

51
internal/config/lambda/target/lazyinit.go Normal file
View File

@@ -0,0 +1,51 @@
// Copyright (c) 2015-2023 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package target
import (
"sync"
"sync/atomic"
)
// Inspired from Golang sync.Once but it is only marked
// initialized when the provided function returns nil.
type lazyInit struct {
done uint32
m sync.Mutex
}
func (l *lazyInit) Do(f func() error) error {
if atomic.LoadUint32(&l.done) == 0 {
return l.doSlow(f)
}
return nil
}
func (l *lazyInit) doSlow(f func() error) error {
l.m.Lock()
defer l.m.Unlock()
if atomic.LoadUint32(&l.done) == 0 {
if err := f(); err != nil {
return err
}
// Mark as done only when f() is successful
atomic.StoreUint32(&l.done, 1)
}
return nil
}

246
internal/config/lambda/target/webhook.go Normal file
View File

@@ -0,0 +1,246 @@
// Copyright (c) 2015-2023 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package target
import (
"bytes"
"context"
"crypto/tls"
"encoding/json"
"errors"
"net/http"
"strings"
"sync/atomic"
"syscall"
"time"
"github.com/minio/minio/internal/config/lambda/event"
xhttp "github.com/minio/minio/internal/http"
"github.com/minio/minio/internal/logger"
"github.com/minio/pkg/v3/certs"
xnet "github.com/minio/pkg/v3/net"
)
// Webhook constants
const (
WebhookEndpoint = "endpoint"
WebhookAuthToken = "auth_token"
WebhookClientCert = "client_cert"
WebhookClientKey = "client_key"
EnvWebhookEnable = "MINIO_LAMBDA_WEBHOOK_ENABLE"
EnvWebhookEndpoint = "MINIO_LAMBDA_WEBHOOK_ENDPOINT"
EnvWebhookAuthToken = "MINIO_LAMBDA_WEBHOOK_AUTH_TOKEN"
EnvWebhookClientCert = "MINIO_LAMBDA_WEBHOOK_CLIENT_CERT"
EnvWebhookClientKey = "MINIO_LAMBDA_WEBHOOK_CLIENT_KEY"
)
// WebhookArgs - Webhook target arguments.
type WebhookArgs struct {
Enable bool `json:"enable"`
Endpoint xnet.URL `json:"endpoint"`
AuthToken string `json:"authToken"`
Transport *http.Transport `json:"-"`
ClientCert string `json:"clientCert"`
ClientKey string `json:"clientKey"`
}
// Validate WebhookArgs fields
func (w WebhookArgs) Validate() error {
if !w.Enable {
return nil
}
if w.Endpoint.IsEmpty() {
return errors.New("endpoint empty")
}
if w.ClientCert != "" && w.ClientKey == "" || w.ClientCert == "" && w.ClientKey != "" {
return errors.New("cert and key must be specified as a pair")
}
return nil
}
// WebhookTarget - Webhook target.
type WebhookTarget struct {
activeRequests int64
totalRequests int64
failedRequests int64
lazyInit lazyInit
id event.TargetID
args WebhookArgs
transport *http.Transport
httpClient *http.Client
loggerOnce logger.LogOnce
cancel context.CancelFunc
cancelCh <-chan struct{}
}
// ID - returns target ID.
func (target *WebhookTarget) ID() event.TargetID {
return target.id
}
// IsActive - Return true if target is up and active
func (target *WebhookTarget) IsActive() (bool, error) {
if err := target.init(); err != nil {
return false, err
}
return target.isActive()
}
// errNotConnected - indicates that the target connection is not active.
var errNotConnected = errors.New("not connected to target server/service")
func (target *WebhookTarget) isActive() (bool, error) {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
req, err := http.NewRequestWithContext(ctx, http.MethodHead, target.args.Endpoint.String(), nil)
if err != nil {
if xnet.IsNetworkOrHostDown(err, false) {
return false, errNotConnected
}
return false, err
}
tokens := strings.Fields(target.args.AuthToken)
switch len(tokens) {
case 2:
req.Header.Set("Authorization", target.args.AuthToken)
case 1:
req.Header.Set("Authorization", "Bearer "+target.args.AuthToken)
}
resp, err := target.httpClient.Do(req)
if err != nil {
if xnet.IsNetworkOrHostDown(err, true) {
return false, errNotConnected
}
return false, err
}
xhttp.DrainBody(resp.Body)
// No network failure i.e response from the target means its up
return true, nil
}
// Stat - returns lambda webhook target statistics such as
// current calls in progress, successfully completed functions
// failed functions.
func (target *WebhookTarget) Stat() event.TargetStat {
return event.TargetStat{
ID: target.id,
ActiveRequests: atomic.LoadInt64(&target.activeRequests),
TotalRequests: atomic.LoadInt64(&target.totalRequests),
FailedRequests: atomic.LoadInt64(&target.failedRequests),
}
}
// Send - sends an event to the webhook.
func (target *WebhookTarget) Send(eventData event.Event) (resp *http.Response, err error) {
atomic.AddInt64(&target.activeRequests, 1)
defer atomic.AddInt64(&target.activeRequests, -1)
atomic.AddInt64(&target.totalRequests, 1)
defer func() {
if err != nil {
atomic.AddInt64(&target.failedRequests, 1)
}
}()
if err = target.init(); err != nil {
return nil, err
}
data, err := json.Marshal(eventData)
if err != nil {
return nil, err
}
req, err := http.NewRequest(http.MethodPost, target.args.Endpoint.String(), bytes.NewReader(data))
if err != nil {
return nil, err
}
// Verify if the authToken already contains
// <Key> <Token> like format, if this is
// already present we can blindly use the
// authToken as is instead of adding 'Bearer'
tokens := strings.Fields(target.args.AuthToken)
switch len(tokens) {
case 2:
req.Header.Set("Authorization", target.args.AuthToken)
case 1:
req.Header.Set("Authorization", "Bearer "+target.args.AuthToken)
}
req.Header.Set("Content-Type", "application/json")
return target.httpClient.Do(req)
}
// Close the target. Will cancel all active requests.
func (target *WebhookTarget) Close() error {
target.cancel()
return nil
}
func (target *WebhookTarget) init() error {
return target.lazyInit.Do(target.initWebhook)
}
// Only called from init()
func (target *WebhookTarget) initWebhook() error {
args := target.args
transport := target.transport
if args.ClientCert != "" && args.ClientKey != "" {
manager, err := certs.NewManager(context.Background(), args.ClientCert, args.ClientKey, tls.LoadX509KeyPair)
if err != nil {
return err
}
manager.ReloadOnSignal(syscall.SIGHUP) // allow reloads upon SIGHUP
transport.TLSClientConfig.GetClientCertificate = manager.GetClientCertificate
}
target.httpClient = &http.Client{Transport: transport}
yes, err := target.isActive()
if err != nil {
return err
}
if !yes {
return errNotConnected
}
return nil
}
// NewWebhookTarget - creates new Webhook target.
func NewWebhookTarget(ctx context.Context, id string, args WebhookArgs, loggerOnce logger.LogOnce, transport *http.Transport) (*WebhookTarget, error) {
ctx, cancel := context.WithCancel(ctx)
target := &WebhookTarget{
id: event.TargetID{ID: id, Name: "webhook"},
args: args,
loggerOnce: loggerOnce,
transport: transport,
cancel: cancel,
cancelCh: ctx.Done(),
}
return target, nil
}

33
internal/config/legacy.go Normal file
View File

@@ -0,0 +1,33 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package config
// One time migration code section
// SetRegion - One time migration code needed, for migrating from older config to new for server Region.
func SetRegion(c Config, name string) {
if name == "" {
return
}
c[RegionSubSys][Default] = KVS{
KV{
Key: RegionName,
Value: name,
},
}
}

69
internal/config/notify/config.go Normal file
View File

@@ -0,0 +1,69 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package notify
import (
"github.com/minio/minio/internal/event/target"
)
// Config - notification target configuration structure, holds
// information about various notification targets.
type Config struct {
AMQP map[string]target.AMQPArgs `json:"amqp"`
Elasticsearch map[string]target.ElasticsearchArgs `json:"elasticsearch"`
Kafka map[string]target.KafkaArgs `json:"kafka"`
MQTT map[string]target.MQTTArgs `json:"mqtt"`
MySQL map[string]target.MySQLArgs `json:"mysql"`
NATS map[string]target.NATSArgs `json:"nats"`
NSQ map[string]target.NSQArgs `json:"nsq"`
PostgreSQL map[string]target.PostgreSQLArgs `json:"postgresql"`
Redis map[string]target.RedisArgs `json:"redis"`
Webhook map[string]target.WebhookArgs `json:"webhook"`
}
const (
defaultTarget = "1"
)
// NewConfig - initialize notification config.
func NewConfig() Config {
// Make sure to initialize notification targets
cfg := Config{
NSQ: make(map[string]target.NSQArgs),
AMQP: make(map[string]target.AMQPArgs),
MQTT: make(map[string]target.MQTTArgs),
NATS: make(map[string]target.NATSArgs),
Redis: make(map[string]target.RedisArgs),
MySQL: make(map[string]target.MySQLArgs),
Kafka: make(map[string]target.KafkaArgs),
Webhook: make(map[string]target.WebhookArgs),
PostgreSQL: make(map[string]target.PostgreSQLArgs),
Elasticsearch: make(map[string]target.ElasticsearchArgs),
}
cfg.NSQ[defaultTarget] = target.NSQArgs{}
cfg.AMQP[defaultTarget] = target.AMQPArgs{}
cfg.MQTT[defaultTarget] = target.MQTTArgs{}
cfg.NATS[defaultTarget] = target.NATSArgs{}
cfg.Redis[defaultTarget] = target.RedisArgs{}
cfg.MySQL[defaultTarget] = target.MySQLArgs{}
cfg.Kafka[defaultTarget] = target.KafkaArgs{}
cfg.Webhook[defaultTarget] = target.WebhookArgs{}
cfg.PostgreSQL[defaultTarget] = target.PostgreSQLArgs{}
cfg.Elasticsearch[defaultTarget] = target.ElasticsearchArgs{}
return cfg
}

714
internal/config/notify/help.go Normal file
View File

@@ -0,0 +1,714 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package notify
import (
"github.com/minio/minio/internal/config"
"github.com/minio/minio/internal/event/target"
)
const (
formatComment = `'namespace' reflects current bucket/object list and 'access' reflects a journal of object operations, defaults to 'namespace'`
queueDirComment = `staging dir for undelivered messages e.g. '/home/events'`
queueLimitComment = `maximum limit for undelivered messages, defaults to '100000'`
)
// Help template inputs for all notification targets
var (
HelpWebhook = config.HelpKVS{
config.HelpKV{
Key: target.WebhookEndpoint,
Description: "webhook server endpoint e.g. http://localhost:8080/minio/events",
Type: "url",
Sensitive: true,
},
config.HelpKV{
Key: target.WebhookAuthToken,
Description: "opaque string or JWT authorization token",
Optional: true,
Type: "string",
Sensitive: true,
Secret: true,
},
config.HelpKV{
Key: target.WebhookQueueDir,
Description: queueDirComment,
Optional: true,
Type: "path",
},
config.HelpKV{
Key: target.WebhookQueueLimit,
Description: queueLimitComment,
Optional: true,
Type: "number",
},
config.HelpKV{
Key: config.Comment,
Description: config.DefaultComment,
Optional: true,
Type: "sentence",
},
config.HelpKV{
Key: target.WebhookClientCert,
Description: "client cert for Webhook mTLS auth",
Optional: true,
Type: "string",
Sensitive: true,
},
config.HelpKV{
Key: target.WebhookClientKey,
Description: "client cert key for Webhook mTLS auth",
Optional: true,
Type: "string",
Sensitive: true,
},
}
HelpAMQP = config.HelpKVS{
config.HelpKV{
Key: target.AmqpURL,
Description: "AMQP server endpoint e.g. `amqp://myuser:mypassword@localhost:5672`",
Type: "url",
Sensitive: true,
},
config.HelpKV{
Key: target.AmqpExchange,
Description: "name of the AMQP exchange",
Optional: true,
Type: "string",
},
config.HelpKV{
Key: target.AmqpExchangeType,
Description: "AMQP exchange type",
Optional: true,
Type: "string",
},
config.HelpKV{
Key: target.AmqpRoutingKey,
Description: "routing key for publishing",
Optional: true,
Type: "string",
Sensitive: true,
},
config.HelpKV{
Key: target.AmqpMandatory,
Description: "quietly ignore undelivered messages when set to 'off', default is 'on'",
Optional: true,
Type: "on|off",
},
config.HelpKV{
Key: target.AmqpDurable,
Description: "persist queue across broker restarts when set to 'on', default is 'off'",
Optional: true,
Type: "on|off",
},
config.HelpKV{
Key: target.AmqpNoWait,
Description: "non-blocking message delivery when set to 'on', default is 'off'",
Optional: true,
Type: "on|off",
},
config.HelpKV{
Key: target.AmqpInternal,
Description: "set to 'on' for exchange to be not used directly by publishers, but only when bound to other exchanges",
Optional: true,
Type: "on|off",
},
config.HelpKV{
Key: target.AmqpAutoDeleted,
Description: "auto delete queue when set to 'on', when there are no consumers",
Optional: true,
Type: "on|off",
},
config.HelpKV{
Key: target.AmqpDeliveryMode,
Description: "set to '1' for non-persistent or '2' for persistent queue",
Optional: true,
Type: "number",
},
config.HelpKV{
Key: target.AmqpPublisherConfirms,
Description: "enable consumer acknowledgement and publisher confirms, use this along with queue_dir for guaranteed delivery of all events",
Optional: true,
Type: "on|off",
},
config.HelpKV{
Key: target.AmqpQueueDir,
Description: queueDirComment,
Optional: true,
Type: "path",
},
config.HelpKV{
Key: target.AmqpQueueLimit,
Description: queueLimitComment,
Optional: true,
Type: "number",
},
config.HelpKV{
Key: config.Comment,
Description: config.DefaultComment,
Optional: true,
Type: "sentence",
},
}
HelpKafka = config.HelpKVS{
config.HelpKV{
Key: target.KafkaBrokers,
Description: "comma separated list of Kafka broker addresses",
Type: "csv",
},
config.HelpKV{
Key: target.KafkaTopic,
Description: "Kafka topic used for bucket notifications",
Optional: true,
Type: "string",
},
config.HelpKV{
Key: target.KafkaSASLUsername,
Description: "username for SASL/PLAIN or SASL/SCRAM authentication",
Optional: true,
Type: "string",
Sensitive: true,
},
config.HelpKV{
Key: target.KafkaSASLPassword,
Description: "password for SASL/PLAIN or SASL/SCRAM authentication",
Optional: true,
Type: "string",
Sensitive: true,
Secret: true,
},
config.HelpKV{
Key: target.KafkaSASLMechanism,
Description: "sasl authentication mechanism, default 'plain'",
Optional: true,
Type: "string",
},
config.HelpKV{
Key: target.KafkaTLSClientAuth,
Description: "clientAuth determines the Kafka server's policy for TLS client auth",
Optional: true,
Type: "string",
},
config.HelpKV{
Key: target.KafkaSASL,
Description: "set to 'on' to enable SASL authentication",
Optional: true,
Type: "on|off",
},
config.HelpKV{
Key: target.KafkaTLS,
Description: "set to 'on' to enable TLS",
Optional: true,
Type: "on|off",
},
config.HelpKV{
Key: target.KafkaTLSSkipVerify,
Description: `trust server TLS without verification, defaults to "on" (verify)`,
Optional: true,
Type: "on|off",
},
config.HelpKV{
Key: target.KafkaClientTLSCert,
Description: "path to client certificate for mTLS auth",
Optional: true,
Type: "path",
Sensitive: true,
},
config.HelpKV{
Key: target.KafkaClientTLSKey,
Description: "path to client key for mTLS auth",
Optional: true,
Type: "path",
Sensitive: true,
},
config.HelpKV{
Key: target.KafkaQueueDir,
Description: queueDirComment,
Optional: true,
Type: "path",
},
config.HelpKV{
Key: target.KafkaQueueLimit,
Description: queueLimitComment,
Optional: true,
Type: "number",
},
config.HelpKV{
Key: target.KafkaVersion,
Description: "specify the version of the Kafka cluster",
Optional: true,
Type: "string",
},
config.HelpKV{
Key: config.Comment,
Description: config.DefaultComment,
Optional: true,
Type: "sentence",
},
config.HelpKV{
Key: target.KafkaCompressionCodec,
Description: "specify compression_codec of the Kafka cluster",
Optional: true,
Type: "none|snappy|gzip|lz4|zstd",
},
config.HelpKV{
Key: target.KafkaCompressionLevel,
Description: "specify compression level of the Kafka cluster",
Optional: true,
Type: "number",
},
config.HelpKV{
Key: target.KafkaBatchSize,
Description: "batch size of the events; used only when queue_dir is set",
Optional: true,
Type: "number",
},
config.HelpKV{
Key: target.KafkaBatchCommitTimeout,
Description: "commit timeout set for the batch; used only when batch_size > 1",
Optional: true,
Type: "duration",
},
}
HelpMQTT = config.HelpKVS{
config.HelpKV{
Key: target.MqttBroker,
Description: "MQTT server endpoint e.g. `tcp://localhost:1883`",
Type: "uri",
Sensitive: true,
},
config.HelpKV{
Key: target.MqttTopic,
Description: "name of the MQTT topic to publish",
Type: "string",
},
config.HelpKV{
Key: target.MqttUsername,
Description: "MQTT username",
Optional: true,
Type: "string",
Sensitive: true,
},
config.HelpKV{
Key: target.MqttPassword,
Description: "MQTT password",
Optional: true,
Type: "string",
Sensitive: true,
Secret: true,
},
config.HelpKV{
Key: target.MqttQoS,
Description: "set the quality of service priority, defaults to '0'",
Optional: true,
Type: "number",
},
config.HelpKV{
Key: target.MqttKeepAliveInterval,
Description: "keep-alive interval for MQTT connections in s,m,h,d",
Optional: true,
Type: "duration",
},
config.HelpKV{
Key: target.MqttReconnectInterval,
Description: "reconnect interval for MQTT connections in s,m,h,d",
Optional: true,
Type: "duration",
},
config.HelpKV{
Key: target.MqttQueueDir,
Description: queueDirComment,
Optional: true,
Type: "path",
},
config.HelpKV{
Key: target.MqttQueueLimit,
Description: queueLimitComment,
Optional: true,
Type: "number",
},
config.HelpKV{
Key: config.Comment,
Description: config.DefaultComment,
Optional: true,
Type: "sentence",
},
}
HelpPostgres = config.HelpKVS{
config.HelpKV{
Key: target.PostgresConnectionString,
Description: `Postgres server connection-string e.g. "host=localhost port=5432 dbname=minio_events user=postgres password=password sslmode=disable"`,
Type: "string",
Sensitive: true,
},
config.HelpKV{
Key: target.PostgresTable,
Description: "DB table name to store/update events, table is auto-created",
Type: "string",
},
config.HelpKV{
Key: target.PostgresFormat,
Description: formatComment,
Type: "namespace*|access",
},
config.HelpKV{
Key: target.PostgresQueueDir,
Description: queueDirComment,
Optional: true,
Type: "path",
},
config.HelpKV{
Key: target.PostgresQueueLimit,
Description: queueLimitComment,
Optional: true,
Type: "number",
},
config.HelpKV{
Key: config.Comment,
Description: config.DefaultComment,
Optional: true,
Type: "sentence",
},
config.HelpKV{
Key: target.PostgresMaxOpenConnections,
Description: "To set the maximum number of open connections to the database. The value is set to `2` by default.",
Optional: true,
Type: "number",
},
}
HelpMySQL = config.HelpKVS{
config.HelpKV{
Key: target.MySQLDSNString,
Description: `MySQL data-source-name connection string e.g. "<user>:<password>@tcp(<host>:<port>)/<database>"`,
Optional: true,
Type: "string",
Sensitive: true,
},
config.HelpKV{
Key: target.MySQLTable,
Description: "DB table name to store/update events, table is auto-created",
Type: "string",
},
config.HelpKV{
Key: target.MySQLFormat,
Description: formatComment,
Type: "namespace*|access",
},
config.HelpKV{
Key: target.MySQLQueueDir,
Description: queueDirComment,
Optional: true,
Type: "path",
},
config.HelpKV{
Key: target.MySQLQueueLimit,
Description: queueLimitComment,
Optional: true,
Type: "number",
},
config.HelpKV{
Key: config.Comment,
Description: config.DefaultComment,
Optional: true,
Type: "sentence",
},
config.HelpKV{
Key: target.MySQLMaxOpenConnections,
Description: "To set the maximum number of open connections to the database. The value is set to `2` by default.",
Optional: true,
Type: "number",
},
}
HelpNATS = config.HelpKVS{
config.HelpKV{
Key: target.NATSAddress,
Description: "NATS server address e.g. '0.0.0.0:4222'",
Type: "address",
Sensitive: true,
},
config.HelpKV{
Key: target.NATSSubject,
Description: "NATS subscription subject",
Type: "string",
},
config.HelpKV{
Key: target.NATSUsername,
Description: "NATS username",
Optional: true,
Type: "string",
Sensitive: true,
},
config.HelpKV{
Key: target.NATSPassword,
Description: "NATS password",
Optional: true,
Type: "string",
Sensitive: true,
Secret: true,
},
config.HelpKV{
Key: target.NATSToken,
Description: "NATS token",
Optional: true,
Type: "string",
Sensitive: true,
Secret: true,
},
config.HelpKV{
Key: target.NATSTLS,
Description: "set to 'on' to enable TLS",
Optional: true,
Type: "on|off",
},
config.HelpKV{
Key: target.NATSTLSSkipVerify,
Description: `trust server TLS without verification, defaults to "on" (verify)`,
Optional: true,
Type: "on|off",
},
config.HelpKV{
Key: target.NATSPingInterval,
Description: "client ping commands interval in s,m,h,d. Disabled by default",
Optional: true,
Type: "duration",
},
config.HelpKV{
Key: target.NATSCertAuthority,
Description: "path to certificate chain of the target NATS server",
Optional: true,
Type: "string",
Sensitive: true,
},
config.HelpKV{
Key: target.NATSClientCert,
Description: "client cert for NATS mTLS auth",
Optional: true,
Type: "string",
Sensitive: true,
},
config.HelpKV{
Key: target.NATSClientKey,
Description: "client cert key for NATS mTLS auth",
Optional: true,
Type: "string",
Sensitive: true,
},
config.HelpKV{
Key: target.NATSJetStream,
Description: "enable JetStream support",
Optional: true,
Type: "on|off",
},
config.HelpKV{
Key: target.NATSQueueDir,
Description: queueDirComment,
Optional: true,
Type: "path",
},
config.HelpKV{
Key: target.NATSQueueLimit,
Description: queueLimitComment,
Optional: true,
Type: "number",
},
config.HelpKV{
Key: target.NATSStreaming,
Description: "[DEPRECATED] set to 'on', to use streaming NATS server",
Optional: true,
Type: "on|off",
},
config.HelpKV{
Key: target.NATSStreamingAsync,
Description: "[DEPRECATED] set to 'on', to enable asynchronous publish",
Optional: true,
Type: "on|off",
},
config.HelpKV{
Key: target.NATSStreamingMaxPubAcksInFlight,
Description: "[DEPRECATED] number of messages to publish without waiting for ACKs",
Optional: true,
Type: "number",
},
config.HelpKV{
Key: target.NATSStreamingClusterID,
Description: "[DEPRECATED] unique ID for NATS streaming cluster",
Optional: true,
Type: "string",
},
config.HelpKV{
Key: config.Comment,
Description: config.DefaultComment,
Optional: true,
Type: "sentence",
},
}
HelpNSQ = config.HelpKVS{
config.HelpKV{
Key: target.NSQAddress,
Description: "NSQ server address e.g. '127.0.0.1:4150'",
Type: "address",
Sensitive: true,
},
config.HelpKV{
Key: target.NSQTopic,
Description: "NSQ topic",
Type: "string",
},
config.HelpKV{
Key: target.NSQTLS,
Description: "set to 'on' to enable TLS",
Optional: true,
Type: "on|off",
},
config.HelpKV{
Key: target.NSQTLSSkipVerify,
Description: `trust server TLS without verification, defaults to "on" (verify)`,
Optional: true,
Type: "on|off",
},
config.HelpKV{
Key: target.NSQQueueDir,
Description: queueDirComment,
Optional: true,
Type: "path",
},
config.HelpKV{
Key: target.NSQQueueLimit,
Description: queueLimitComment,
Optional: true,
Type: "number",
},
config.HelpKV{
Key: config.Comment,
Description: config.DefaultComment,
Optional: true,
Type: "sentence",
},
}
HelpES = config.HelpKVS{
config.HelpKV{
Key: target.ElasticURL,
Description: "Elasticsearch server's address, with optional authentication info",
Type: "url",
Sensitive: true,
},
config.HelpKV{
Key: target.ElasticIndex,
Description: `Elasticsearch index to store/update events, index is auto-created`,
Type: "string",
},
config.HelpKV{
Key: target.ElasticFormat,
Description: formatComment,
Type: "namespace*|access",
},
config.HelpKV{
Key: target.ElasticQueueDir,
Description: queueDirComment,
Optional: true,
Type: "path",
},
config.HelpKV{
Key: target.ElasticQueueLimit,
Description: queueLimitComment,
Optional: true,
Type: "number",
},
config.HelpKV{
Key: target.ElasticUsername,
Description: "username for Elasticsearch basic-auth",
Optional: true,
Type: "string",
Sensitive: true,
},
config.HelpKV{
Key: target.ElasticPassword,
Description: "password for Elasticsearch basic-auth",
Optional: true,
Type: "string",
Sensitive: true,
Secret: true,
},
config.HelpKV{
Key: config.Comment,
Description: config.DefaultComment,
Optional: true,
Type: "sentence",
},
}
HelpRedis = config.HelpKVS{
config.HelpKV{
Key: target.RedisAddress,
Description: "Redis server's address. For example: `localhost:6379`",
Type: "address",
Sensitive: true,
},
config.HelpKV{
Key: target.RedisKey,
Description: "Redis key to store/update events, key is auto-created",
Type: "string",
Sensitive: true,
},
config.HelpKV{
Key: target.RedisFormat,
Description: formatComment,
Type: "namespace*|access",
},
config.HelpKV{
Key: target.RedisPassword,
Description: "Redis server password",
Optional: true,
Type: "string",
Sensitive: true,
Secret: true,
},
config.HelpKV{
Key: target.RedisUser,
Description: "Redis server user for the auth",
Optional: true,
Type: "string",
},
config.HelpKV{
Key: target.RedisQueueDir,
Description: queueDirComment,
Optional: true,
Type: "path",
},
config.HelpKV{
Key: target.RedisQueueLimit,
Description: queueLimitComment,
Optional: true,
Type: "number",
},
config.HelpKV{
Key: config.Comment,
Description: config.DefaultComment,
Optional: true,
Type: "sentence",
},
}
)

649
internal/config/notify/legacy.go Normal file
View File

@@ -0,0 +1,649 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package notify
import (
"fmt"
"strconv"
"strings"
"github.com/minio/minio/internal/config"
"github.com/minio/minio/internal/event/target"
)
// SetNotifyKafka - helper for config migration from older config.
func SetNotifyKafka(s config.Config, name string, cfg target.KafkaArgs) error {
if !cfg.Enable {
return nil
}
if err := cfg.Validate(); err != nil {
return err
}
s[config.NotifyKafkaSubSys][name] = config.KVS{
config.KV{
Key: config.Enable,
Value: config.EnableOn,
},
config.KV{
Key: target.KafkaBrokers,
Value: func() string {
var brokers []string
for _, broker := range cfg.Brokers {
brokers = append(brokers, broker.String())
}
return strings.Join(brokers, config.ValueSeparator)
}(),
},
config.KV{
Key: target.KafkaTopic,
Value: cfg.Topic,
},
config.KV{
Key: target.KafkaQueueDir,
Value: cfg.QueueDir,
},
config.KV{
Key: target.KafkaClientTLSCert,
Value: cfg.TLS.ClientTLSCert,
},
config.KV{
Key: target.KafkaClientTLSKey,
Value: cfg.TLS.ClientTLSKey,
},
config.KV{
Key: target.KafkaQueueLimit,
Value: strconv.Itoa(int(cfg.QueueLimit)),
},
config.KV{
Key: target.KafkaTLS,
Value: config.FormatBool(cfg.TLS.Enable),
},
config.KV{
Key: target.KafkaTLSSkipVerify,
Value: config.FormatBool(cfg.TLS.SkipVerify),
},
config.KV{
Key: target.KafkaTLSClientAuth,
Value: strconv.Itoa(int(cfg.TLS.ClientAuth)),
},
config.KV{
Key: target.KafkaSASL,
Value: config.FormatBool(cfg.SASL.Enable),
},
config.KV{
Key: target.KafkaSASLUsername,
Value: cfg.SASL.User,
},
config.KV{
Key: target.KafkaSASLPassword,
Value: cfg.SASL.Password,
},
config.KV{
Key: target.KafkaCompressionCodec,
Value: cfg.Producer.Compression,
},
config.KV{
Key: target.KafkaCompressionLevel,
Value: strconv.Itoa(cfg.Producer.CompressionLevel),
},
}
return nil
}
// SetNotifyAMQP - helper for config migration from older config.
func SetNotifyAMQP(s config.Config, amqpName string, cfg target.AMQPArgs) error {
if !cfg.Enable {
return nil
}
if err := cfg.Validate(); err != nil {
return err
}
s[config.NotifyAMQPSubSys][amqpName] = config.KVS{
config.KV{
Key: config.Enable,
Value: config.EnableOn,
},
config.KV{
Key: target.AmqpURL,
Value: cfg.URL.String(),
},
config.KV{
Key: target.AmqpExchange,
Value: cfg.Exchange,
},
config.KV{
Key: target.AmqpRoutingKey,
Value: cfg.RoutingKey,
},
config.KV{
Key: target.AmqpExchangeType,
Value: cfg.ExchangeType,
},
config.KV{
Key: target.AmqpDeliveryMode,
Value: strconv.Itoa(int(cfg.DeliveryMode)),
},
config.KV{
Key: target.AmqpMandatory,
Value: config.FormatBool(cfg.Mandatory),
},
config.KV{
Key: target.AmqpInternal,
Value: config.FormatBool(cfg.Immediate),
},
config.KV{
Key: target.AmqpDurable,
Value: config.FormatBool(cfg.Durable),
},
config.KV{
Key: target.AmqpNoWait,
Value: config.FormatBool(cfg.NoWait),
},
config.KV{
Key: target.AmqpAutoDeleted,
Value: config.FormatBool(cfg.AutoDeleted),
},
config.KV{
Key: target.AmqpQueueDir,
Value: cfg.QueueDir,
},
config.KV{
Key: target.AmqpQueueLimit,
Value: strconv.Itoa(int(cfg.QueueLimit)),
},
}
return nil
}
// SetNotifyES - helper for config migration from older config.
func SetNotifyES(s config.Config, esName string, cfg target.ElasticsearchArgs) error {
if !cfg.Enable {
return nil
}
if err := cfg.Validate(); err != nil {
return err
}
s[config.NotifyESSubSys][esName] = config.KVS{
config.KV{
Key: config.Enable,
Value: config.EnableOn,
},
config.KV{
Key: target.ElasticFormat,
Value: cfg.Format,
},
config.KV{
Key: target.ElasticURL,
Value: cfg.URL.String(),
},
config.KV{
Key: target.ElasticIndex,
Value: cfg.Index,
},
config.KV{
Key: target.ElasticQueueDir,
Value: cfg.QueueDir,
},
config.KV{
Key: target.ElasticQueueLimit,
Value: strconv.Itoa(int(cfg.QueueLimit)),
},
config.KV{
Key: target.ElasticUsername,
Value: cfg.Username,
},
config.KV{
Key: target.ElasticPassword,
Value: cfg.Password,
},
}
return nil
}
// SetNotifyRedis - helper for config migration from older config.
func SetNotifyRedis(s config.Config, redisName string, cfg target.RedisArgs) error {
if !cfg.Enable {
return nil
}
if err := cfg.Validate(); err != nil {
return err
}
s[config.NotifyRedisSubSys][redisName] = config.KVS{
config.KV{
Key: config.Enable,
Value: config.EnableOn,
},
config.KV{
Key: target.RedisFormat,
Value: cfg.Format,
},
config.KV{
Key: target.RedisAddress,
Value: cfg.Addr.String(),
},
config.KV{
Key: target.RedisPassword,
Value: cfg.Password,
},
config.KV{
Key: target.RedisUser,
Value: cfg.User,
},
config.KV{
Key: target.RedisKey,
Value: cfg.Key,
},
config.KV{
Key: target.RedisQueueDir,
Value: cfg.QueueDir,
},
config.KV{
Key: target.RedisQueueLimit,
Value: strconv.Itoa(int(cfg.QueueLimit)),
},
}
return nil
}
// SetNotifyWebhook - helper for config migration from older config.
func SetNotifyWebhook(s config.Config, whName string, cfg target.WebhookArgs) error {
if !cfg.Enable {
return nil
}
if err := cfg.Validate(); err != nil {
return err
}
s[config.NotifyWebhookSubSys][whName] = config.KVS{
config.KV{
Key: config.Enable,
Value: config.EnableOn,
},
config.KV{
Key: target.WebhookEndpoint,
Value: cfg.Endpoint.String(),
},
config.KV{
Key: target.WebhookAuthToken,
Value: cfg.AuthToken,
},
config.KV{
Key: target.WebhookQueueDir,
Value: cfg.QueueDir,
},
config.KV{
Key: target.WebhookQueueLimit,
Value: strconv.Itoa(int(cfg.QueueLimit)),
},
config.KV{
Key: target.WebhookClientCert,
Value: cfg.ClientCert,
},
config.KV{
Key: target.WebhookClientKey,
Value: cfg.ClientKey,
},
}
return nil
}
// SetNotifyPostgres - helper for config migration from older config.
func SetNotifyPostgres(s config.Config, psqName string, cfg target.PostgreSQLArgs) error {
if !cfg.Enable {
return nil
}
if err := cfg.Validate(); err != nil {
return err
}
s[config.NotifyPostgresSubSys][psqName] = config.KVS{
config.KV{
Key: config.Enable,
Value: config.EnableOn,
},
config.KV{
Key: target.PostgresFormat,
Value: cfg.Format,
},
config.KV{
Key: target.PostgresConnectionString,
Value: cfg.ConnectionString,
},
config.KV{
Key: target.PostgresTable,
Value: cfg.Table,
},
config.KV{
Key: target.PostgresHost,
Value: cfg.Host.String(),
},
config.KV{
Key: target.PostgresPort,
Value: cfg.Port,
},
config.KV{
Key: target.PostgresUsername,
Value: cfg.Username,
},
config.KV{
Key: target.PostgresPassword,
Value: cfg.Password,
},
config.KV{
Key: target.PostgresDatabase,
Value: cfg.Database,
},
config.KV{
Key: target.PostgresQueueDir,
Value: cfg.QueueDir,
},
config.KV{
Key: target.PostgresQueueLimit,
Value: strconv.Itoa(int(cfg.QueueLimit)),
},
config.KV{
Key: target.PostgresMaxOpenConnections,
Value: strconv.Itoa(cfg.MaxOpenConnections),
},
}
return nil
}
// SetNotifyNSQ - helper for config migration from older config.
func SetNotifyNSQ(s config.Config, nsqName string, cfg target.NSQArgs) error {
if !cfg.Enable {
return nil
}
if err := cfg.Validate(); err != nil {
return err
}
s[config.NotifyNSQSubSys][nsqName] = config.KVS{
config.KV{
Key: config.Enable,
Value: config.EnableOn,
},
config.KV{
Key: target.NSQAddress,
Value: cfg.NSQDAddress.String(),
},
config.KV{
Key: target.NSQTopic,
Value: cfg.Topic,
},
config.KV{
Key: target.NSQTLS,
Value: config.FormatBool(cfg.TLS.Enable),
},
config.KV{
Key: target.NSQTLSSkipVerify,
Value: config.FormatBool(cfg.TLS.SkipVerify),
},
config.KV{
Key: target.NSQQueueDir,
Value: cfg.QueueDir,
},
config.KV{
Key: target.NSQQueueLimit,
Value: strconv.Itoa(int(cfg.QueueLimit)),
},
}
return nil
}
// SetNotifyNATS - helper for config migration from older config.
func SetNotifyNATS(s config.Config, natsName string, cfg target.NATSArgs) error {
if !cfg.Enable {
return nil
}
if err := cfg.Validate(); err != nil {
return err
}
s[config.NotifyNATSSubSys][natsName] = config.KVS{
config.KV{
Key: config.Enable,
Value: config.EnableOn,
},
config.KV{
Key: target.NATSAddress,
Value: cfg.Address.String(),
},
config.KV{
Key: target.NATSSubject,
Value: cfg.Subject,
},
config.KV{
Key: target.NATSUsername,
Value: cfg.Username,
},
config.KV{
Key: target.NATSUserCredentials,
Value: cfg.UserCredentials,
},
config.KV{
Key: target.NATSPassword,
Value: cfg.Password,
},
config.KV{
Key: target.NATSToken,
Value: cfg.Token,
},
config.KV{
Key: target.NATSNKeySeed,
Value: cfg.NKeySeed,
},
config.KV{
Key: target.NATSCertAuthority,
Value: cfg.CertAuthority,
},
config.KV{
Key: target.NATSClientCert,
Value: cfg.ClientCert,
},
config.KV{
Key: target.NATSClientKey,
Value: cfg.ClientKey,
},
config.KV{
Key: target.NATSTLS,
Value: config.FormatBool(cfg.Secure),
},
config.KV{
Key: target.NATSTLSSkipVerify,
Value: config.FormatBool(cfg.Secure),
},
config.KV{
Key: target.NATSTLSHandshakeFirst,
Value: config.FormatBool(cfg.TLSHandshakeFirst),
},
config.KV{
Key: target.NATSPingInterval,
Value: strconv.FormatInt(cfg.PingInterval, 10),
},
config.KV{
Key: target.NATSQueueDir,
Value: cfg.QueueDir,
},
config.KV{
Key: target.NATSQueueLimit,
Value: strconv.Itoa(int(cfg.QueueLimit)),
},
config.KV{
Key: target.NATSStreaming,
Value: func() string {
if cfg.Streaming.Enable {
return config.EnableOn
}
return config.EnableOff
}(),
},
config.KV{
Key: target.NATSStreamingClusterID,
Value: cfg.Streaming.ClusterID,
},
config.KV{
Key: target.NATSStreamingAsync,
Value: config.FormatBool(cfg.Streaming.Async),
},
config.KV{
Key: target.NATSStreamingMaxPubAcksInFlight,
Value: strconv.Itoa(cfg.Streaming.MaxPubAcksInflight),
},
}
return nil
}
// SetNotifyMySQL - helper for config migration from older config.
func SetNotifyMySQL(s config.Config, sqlName string, cfg target.MySQLArgs) error {
if !cfg.Enable {
return nil
}
if err := cfg.Validate(); err != nil {
return err
}
s[config.NotifyMySQLSubSys][sqlName] = config.KVS{
config.KV{
Key: config.Enable,
Value: config.EnableOn,
},
config.KV{
Key: target.MySQLFormat,
Value: cfg.Format,
},
config.KV{
Key: target.MySQLDSNString,
Value: cfg.DSN,
},
config.KV{
Key: target.MySQLTable,
Value: cfg.Table,
},
config.KV{
Key: target.MySQLHost,
Value: cfg.Host.String(),
},
config.KV{
Key: target.MySQLPort,
Value: cfg.Port,
},
config.KV{
Key: target.MySQLUsername,
Value: cfg.User,
},
config.KV{
Key: target.MySQLPassword,
Value: cfg.Password,
},
config.KV{
Key: target.MySQLDatabase,
Value: cfg.Database,
},
config.KV{
Key: target.MySQLQueueDir,
Value: cfg.QueueDir,
},
config.KV{
Key: target.MySQLQueueLimit,
Value: strconv.Itoa(int(cfg.QueueLimit)),
},
config.KV{
Key: target.MySQLMaxOpenConnections,
Value: strconv.Itoa(cfg.MaxOpenConnections),
},
}
return nil
}
// SetNotifyMQTT - helper for config migration from older config.
func SetNotifyMQTT(s config.Config, mqttName string, cfg target.MQTTArgs) error {
if !cfg.Enable {
return nil
}
if err := cfg.Validate(); err != nil {
return err
}
s[config.NotifyMQTTSubSys][mqttName] = config.KVS{
config.KV{
Key: config.Enable,
Value: config.EnableOn,
},
config.KV{
Key: target.MqttBroker,
Value: cfg.Broker.String(),
},
config.KV{
Key: target.MqttTopic,
Value: cfg.Topic,
},
config.KV{
Key: target.MqttQoS,
Value: fmt.Sprintf("%d", cfg.QoS),
},
config.KV{
Key: target.MqttUsername,
Value: cfg.User,
},
config.KV{
Key: target.MqttPassword,
Value: cfg.Password,
},
config.KV{
Key: target.MqttReconnectInterval,
Value: cfg.MaxReconnectInterval.String(),
},
config.KV{
Key: target.MqttKeepAliveInterval,
Value: cfg.KeepAlive.String(),
},
config.KV{
Key: target.MqttQueueDir,
Value: cfg.QueueDir,
},
config.KV{
Key: target.MqttQueueLimit,
Value: strconv.Itoa(int(cfg.QueueLimit)),
},
}
return nil
}

1796
internal/config/notify/parse.go Normal file
View File

File diff suppressed because it is too large Load Diff

230
internal/config/policy/opa/config.go Normal file
View File

@@ -0,0 +1,230 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package opa
import (
"bytes"
"encoding/json"
"io"
"net/http"
"github.com/minio/minio/internal/config"
"github.com/minio/pkg/v3/env"
xnet "github.com/minio/pkg/v3/net"
"github.com/minio/pkg/v3/policy"
)
// Env IAM OPA URL
const (
URL = "url"
AuthToken = "auth_token"
EnvPolicyOpaURL = "MINIO_POLICY_OPA_URL"
EnvPolicyOpaAuthToken = "MINIO_POLICY_OPA_AUTH_TOKEN"
)
// DefaultKVS - default config for OPA config
var (
DefaultKVS = config.KVS{
config.KV{
Key: URL,
Value: "",
HiddenIfEmpty: true,
},
config.KV{
Key: AuthToken,
Value: "",
HiddenIfEmpty: true,
},
}
)
// Args opa general purpose policy engine configuration.
type Args struct {
URL *xnet.URL `json:"url"`
AuthToken string `json:"authToken"`
Transport http.RoundTripper `json:"-"`
CloseRespFn func(r io.ReadCloser) `json:"-"`
}
// Validate - validate opa configuration params.
func (a *Args) Validate() error {
req, err := http.NewRequest(http.MethodPost, a.URL.String(), bytes.NewReader([]byte("")))
if err != nil {
return err
}
req.Header.Set("Content-Type", "application/json")
if a.AuthToken != "" {
req.Header.Set("Authorization", a.AuthToken)
}
client := &http.Client{Transport: a.Transport}
resp, err := client.Do(req)
if err != nil {
return err
}
defer a.CloseRespFn(resp.Body)
return nil
}
// UnmarshalJSON - decodes JSON data.
func (a *Args) UnmarshalJSON(data []byte) error {
// subtype to avoid recursive call to UnmarshalJSON()
type subArgs Args
var so subArgs
if err := json.Unmarshal(data, &so); err != nil {
return err
}
oa := Args(so)
if oa.URL == nil || oa.URL.String() == "" {
*a = oa
return nil
}
*a = oa
return nil
}
// Opa - implements opa policy agent calls.
type Opa struct {
args Args
client *http.Client
}
// Enabled returns if opa is enabled.
func Enabled(kvs config.KVS) bool {
return kvs.Get(URL) != ""
}
// LookupConfig lookup Opa from config, override with any ENVs.
func LookupConfig(kv config.KVS, transport *http.Transport, closeRespFn func(io.ReadCloser)) (Args, error) {
args := Args{}
if err := config.CheckValidKeys(config.PolicyOPASubSys, kv, DefaultKVS); err != nil {
return args, err
}
opaURL := env.Get(EnvIamOpaURL, "")
if opaURL == "" {
opaURL = env.Get(EnvPolicyOpaURL, kv.Get(URL))
if opaURL == "" {
return args, nil
}
}
authToken := env.Get(EnvIamOpaAuthToken, "")
if authToken == "" {
authToken = env.Get(EnvPolicyOpaAuthToken, kv.Get(AuthToken))
}
u, err := xnet.ParseHTTPURL(opaURL)
if err != nil {
return args, err
}
args = Args{
URL: u,
AuthToken: authToken,
Transport: transport,
CloseRespFn: closeRespFn,
}
if err = args.Validate(); err != nil {
return args, err
}
return args, nil
}
// New - initializes opa policy engine connector.
func New(args Args) *Opa {
// No opa args.
if args.URL == nil || args.URL.Scheme == "" && args.AuthToken == "" {
return nil
}
return &Opa{
args: args,
client: &http.Client{Transport: args.Transport},
}
}
// IsAllowed - checks given policy args is allowed to continue the REST API.
func (o *Opa) IsAllowed(args policy.Args) (bool, error) {
if o == nil {
return false, nil
}
// OPA input
body := make(map[string]any)
body["input"] = args
inputBytes, err := json.Marshal(body)
if err != nil {
return false, err
}
req, err := http.NewRequest(http.MethodPost, o.args.URL.String(), bytes.NewReader(inputBytes))
if err != nil {
return false, err
}
req.Header.Set("Content-Type", "application/json")
if o.args.AuthToken != "" {
req.Header.Set("Authorization", o.args.AuthToken)
}
resp, err := o.client.Do(req)
if err != nil {
return false, err
}
defer o.args.CloseRespFn(resp.Body)
// Read the body to be saved later.
opaRespBytes, err := io.ReadAll(resp.Body)
if err != nil {
return false, err
}
// Handle large OPA responses when OPA URL is of
// form http://localhost:8181/v1/data/httpapi/authz
type opaResultAllow struct {
Result struct {
Allow bool `json:"allow"`
} `json:"result"`
}
// Handle simpler OPA responses when OPA URL is of
// form http://localhost:8181/v1/data/httpapi/authz/allow
type opaResult struct {
Result bool `json:"result"`
}
respBody := bytes.NewReader(opaRespBytes)
var result opaResult
if err = json.NewDecoder(respBody).Decode(&result); err != nil {
respBody.Seek(0, 0)
var resultAllow opaResultAllow
if err = json.NewDecoder(respBody).Decode(&resultAllow); err != nil {
return false, err
}
return resultAllow.Result.Allow, nil
}
return result.Result, nil
}

50
internal/config/policy/opa/help.go Normal file
View File

@@ -0,0 +1,50 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package opa
import "github.com/minio/minio/internal/config"
// Help template for OPA policy feature.
var (
defaultHelpPostfix = func(key string) string {
return config.DefaultHelpPostfix(DefaultKVS, key)
}
Help = config.HelpKVS{
config.HelpKV{
Key: URL,
Description: `[DEPRECATED] OPA HTTP(s) endpoint e.g. "http://localhost:8181/v1/data/httpapi/authz/allow"` + defaultHelpPostfix(URL),
Type: "url",
Sensitive: true,
},
config.HelpKV{
Key: AuthToken,
Description: "[DEPRECATED] authorization token for OPA endpoint" + defaultHelpPostfix(AuthToken),
Optional: true,
Type: "string",
Sensitive: true,
Secret: true,
},
config.HelpKV{
Key: config.Comment,
Description: config.DefaultComment,
Optional: true,
Type: "sentence",
},
}
)

46
internal/config/policy/opa/legacy.go Normal file
View File

@@ -0,0 +1,46 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package opa
import (
"github.com/minio/minio/internal/config"
)
// Legacy OPA envs
const (
EnvIamOpaURL = "MINIO_IAM_OPA_URL"
EnvIamOpaAuthToken = "MINIO_IAM_OPA_AUTHTOKEN"
)
// SetPolicyOPAConfig - One time migration code needed, for migrating from older config to new for PolicyOPAConfig.
func SetPolicyOPAConfig(s config.Config, opaArgs Args) {
if opaArgs.URL == nil || opaArgs.URL.String() == "" {
// Do not enable if opaArgs was empty.
return
}
s[config.PolicyOPASubSys][config.Default] = config.KVS{
config.KV{
Key: URL,
Value: opaArgs.URL.String(),
},
config.KV{
Key: AuthToken,
Value: opaArgs.AuthToken,
},
}
}

245
internal/config/policy/plugin/config.go Normal file
View File

@@ -0,0 +1,245 @@
// Copyright (c) 2015-2022 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package plugin
import (
"bytes"
"encoding/json"
"io"
"net/http"
"time"
"github.com/minio/minio/internal/config"
xhttp "github.com/minio/minio/internal/http"
xnet "github.com/minio/pkg/v3/net"
"github.com/minio/pkg/v3/policy"
)
// Authorization Plugin config and env variables
const (
URL = "url"
AuthToken = "auth_token"
EnableHTTP2 = "enable_http2"
EnvPolicyPluginURL = "MINIO_POLICY_PLUGIN_URL"
EnvPolicyPluginAuthToken = "MINIO_POLICY_PLUGIN_AUTH_TOKEN"
EnvPolicyPluginEnableHTTP2 = "MINIO_POLICY_PLUGIN_ENABLE_HTTP2"
)
// DefaultKVS - default config for Authz plugin config
var (
DefaultKVS = config.KVS{
config.KV{
Key: URL,
Value: "",
},
config.KV{
Key: AuthToken,
Value: "",
},
config.KV{
Key: EnableHTTP2,
Value: "off",
},
}
)
// Args for general purpose policy engine configuration.
type Args struct {
URL *xnet.URL `json:"url"`
AuthToken string `json:"authToken"`
Transport http.RoundTripper `json:"-"`
CloseRespFn func(r io.ReadCloser) `json:"-"`
}
// Validate - validate opa configuration params.
func (a *Args) Validate() error {
req, err := http.NewRequest(http.MethodPost, a.URL.String(), bytes.NewReader([]byte("")))
if err != nil {
return err
}
req.Header.Set("Content-Type", "application/json")
if a.AuthToken != "" {
req.Header.Set("Authorization", a.AuthToken)
}
client := &http.Client{Transport: a.Transport}
resp, err := client.Do(req)
if err != nil {
return err
}
defer a.CloseRespFn(resp.Body)
return nil
}
// UnmarshalJSON - decodes JSON data.
func (a *Args) UnmarshalJSON(data []byte) error {
// subtype to avoid recursive call to UnmarshalJSON()
type subArgs Args
var so subArgs
if err := json.Unmarshal(data, &so); err != nil {
return err
}
oa := Args(so)
if oa.URL == nil || oa.URL.String() == "" {
*a = oa
return nil
}
*a = oa
return nil
}
// AuthZPlugin - implements opa policy agent calls.
type AuthZPlugin struct {
args Args
client *http.Client
}
// Enabled returns if AuthZPlugin is enabled.
func Enabled(kvs config.KVS) bool {
return kvs.Get(URL) != ""
}
// LookupConfig lookup AuthZPlugin from config, override with any ENVs.
func LookupConfig(s config.Config, httpSettings xhttp.ConnSettings, closeRespFn func(io.ReadCloser)) (Args, error) {
args := Args{}
if err := s.CheckValidKeys(config.PolicyPluginSubSys, nil); err != nil {
return args, err
}
getCfg := func(cfgParam string) string {
// As parameters are already validated, we skip checking
// if the config param was found.
val, _, _ := s.ResolveConfigParam(config.PolicyPluginSubSys, config.Default, cfgParam, false)
return val
}
pluginURL := getCfg(URL)
if pluginURL == "" {
return args, nil
}
u, err := xnet.ParseHTTPURL(pluginURL)
if err != nil {
return args, err
}
enableHTTP2 := false
if v := getCfg(EnableHTTP2); v != "" {
enableHTTP2, err = config.ParseBool(v)
if err != nil {
return args, err
}
}
httpSettings.EnableHTTP2 = enableHTTP2
transport := httpSettings.NewHTTPTransportWithTimeout(time.Minute)
args = Args{
URL: u,
AuthToken: getCfg(AuthToken),
Transport: transport,
CloseRespFn: closeRespFn,
}
if err = args.Validate(); err != nil {
return args, err
}
return args, nil
}
// New - initializes Authorization Management Plugin.
func New(args Args) *AuthZPlugin {
if args.URL == nil || args.URL.Scheme == "" && args.AuthToken == "" {
return nil
}
return &AuthZPlugin{
args: args,
client: &http.Client{Transport: args.Transport},
}
}
// IsAllowed - checks given policy args is allowed to continue the REST API.
func (o *AuthZPlugin) IsAllowed(args policy.Args) (bool, error) {
if o == nil {
return false, nil
}
// Access Management Plugin Input
body := make(map[string]any)
body["input"] = args
inputBytes, err := json.Marshal(body)
if err != nil {
return false, err
}
req, err := http.NewRequest(http.MethodPost, o.args.URL.String(), bytes.NewReader(inputBytes))
if err != nil {
return false, err
}
req.Header.Set("Content-Type", "application/json")
if o.args.AuthToken != "" {
req.Header.Set("Authorization", o.args.AuthToken)
}
resp, err := o.client.Do(req)
if err != nil {
return false, err
}
defer o.args.CloseRespFn(resp.Body)
// Read the body to be saved later.
opaRespBytes, err := io.ReadAll(resp.Body)
if err != nil {
return false, err
}
// Handle large OPA responses when OPA URL is of
// form http://localhost:8181/v1/data/httpapi/authz
type opaResultAllow struct {
Result struct {
Allow bool `json:"allow"`
} `json:"result"`
}
// Handle simpler OPA responses when OPA URL is of
// form http://localhost:8181/v1/data/httpapi/authz/allow
type opaResult struct {
Result bool `json:"result"`
}
respBody := bytes.NewReader(opaRespBytes)
var result opaResult
if err = json.NewDecoder(respBody).Decode(&result); err != nil {
respBody.Seek(0, 0)
var resultAllow opaResultAllow
if err = json.NewDecoder(respBody).Decode(&resultAllow); err != nil {
return false, err
}
return resultAllow.Result.Allow, nil
}
return result.Result, nil
}

56
internal/config/policy/plugin/help.go Normal file
View File

@@ -0,0 +1,56 @@
// Copyright (c) 2015-2022 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package plugin
import "github.com/minio/minio/internal/config"
// Help template for Access Management Plugin policy feature.
var (
defaultHelpPostfix = func(key string) string {
return config.DefaultHelpPostfix(DefaultKVS, key)
}
Help = config.HelpKVS{
config.HelpKV{
Key: URL,
Description: `plugin hook endpoint (HTTP(S)) e.g. "http://localhost:8181/v1/data/httpapi/authz/allow"` + defaultHelpPostfix(URL),
Type: "url",
Sensitive: true,
},
config.HelpKV{
Key: AuthToken,
Description: "authorization header for plugin hook endpoint" + defaultHelpPostfix(AuthToken),
Optional: true,
Type: "string",
Sensitive: true,
Secret: true,
},
config.HelpKV{
Key: EnableHTTP2,
Description: "Enable experimental HTTP2 support to connect to plugin service" + defaultHelpPostfix(EnableHTTP2),
Optional: true,
Type: "bool",
},
config.HelpKV{
Key: config.Comment,
Description: config.DefaultComment,
Optional: true,
Type: "sentence",
},
}
)

48
internal/config/scanner/help.go Normal file
View File

@@ -0,0 +1,48 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package scanner
import "github.com/minio/minio/internal/config"
var (
defaultHelpPostfix = func(key string) string {
return config.DefaultHelpPostfix(DefaultKVS, key)
}
// Help provides help for config values
Help = config.HelpKVS{
config.HelpKV{
Key: Speed,
Description: `customize scanner speed (default|slowest|slow|fast|fastest)` + defaultHelpPostfix(Speed),
Optional: true,
Type: "string",
},
config.HelpKV{
Key: ExcessVersions,
Description: `alert per object beyond this many versions` + defaultHelpPostfix(ExcessVersions),
Optional: true,
Type: "int",
},
config.HelpKV{
Key: ExcessFolders,
Description: `alert beyond this many sub-folders per folder in an erasure set` + defaultHelpPostfix(ExcessFolders),
Optional: true,
Type: "int",
},
}
)

202
internal/config/scanner/scanner.go Normal file
View File

@@ -0,0 +1,202 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package scanner
import (
"fmt"
"strconv"
"time"
"github.com/minio/minio/internal/config"
"github.com/minio/pkg/v3/env"
)
// Compression environment variables
const (
Speed = "speed"
EnvSpeed = "MINIO_SCANNER_SPEED"
IdleSpeed = "idle_speed"
EnvIdleSpeed = "MINIO_SCANNER_IDLE_SPEED"
ExcessVersions = "alert_excess_versions"
EnvExcessVersions = "MINIO_SCANNER_ALERT_EXCESS_VERSIONS"
ExcessFolders = "alert_excess_folders"
EnvExcessFolders = "MINIO_SCANNER_ALERT_EXCESS_FOLDERS"
// All below are deprecated in October 2022 and
// replaced them with a single speed parameter
Delay = "delay"
MaxWait = "max_wait"
Cycle = "cycle"
EnvDelay = "MINIO_SCANNER_DELAY"
EnvCycle = "MINIO_SCANNER_CYCLE"
EnvDelayLegacy = "MINIO_CRAWLER_DELAY"
EnvMaxWait = "MINIO_SCANNER_MAX_WAIT"
EnvMaxWaitLegacy = "MINIO_CRAWLER_MAX_WAIT"
)
// Config represents the heal settings.
type Config struct {
// Delay is the sleep multiplier.
Delay float64 `json:"delay"`
// Sleep always or based on incoming S3 requests.
IdleMode int32 // 0 => on, 1 => off
// Alert upon this many excess object versions
ExcessVersions int64 // 100
// Alert upon this many excess sub-folders per folder in an erasure set.
ExcessFolders int64 // 50000
// MaxWait is maximum wait time between operations
MaxWait time.Duration
// Cycle is the time.Duration between each scanner cycles
Cycle time.Duration
}
// DefaultKVS - default KV config for heal settings
var DefaultKVS = config.KVS{
config.KV{
Key: Speed,
Value: "default",
},
config.KV{
Key: IdleSpeed,
Value: "",
HiddenIfEmpty: true,
},
config.KV{
Key: ExcessVersions,
Value: "100",
},
config.KV{
Key: ExcessFolders,
Value: "50000",
},
// Deprecated Oct 2022
config.KV{
Key: Delay,
Value: "",
HiddenIfEmpty: true,
},
// Deprecated Oct 2022
config.KV{
Key: MaxWait,
Value: "",
HiddenIfEmpty: true,
},
// Deprecated Oct 2022
config.KV{
Key: Cycle,
Value: "",
HiddenIfEmpty: true,
},
}
// LookupConfig - lookup config and override with valid environment settings if any.
func LookupConfig(kvs config.KVS) (cfg Config, err error) {
cfg = Config{
ExcessVersions: 100,
ExcessFolders: 50000,
IdleMode: 0, // Default is on
}
if err = config.CheckValidKeys(config.ScannerSubSys, kvs, DefaultKVS); err != nil {
return cfg, err
}
excessVersions, err := strconv.ParseInt(env.Get(EnvExcessVersions, kvs.GetWithDefault(ExcessVersions, DefaultKVS)), 10, 64)
if err != nil {
return cfg, err
}
cfg.ExcessVersions = excessVersions
excessFolders, err := strconv.ParseInt(env.Get(EnvExcessFolders, kvs.GetWithDefault(ExcessFolders, DefaultKVS)), 10, 64)
if err != nil {
return cfg, err
}
cfg.ExcessFolders = excessFolders
switch idleSpeed := env.Get(EnvIdleSpeed, kvs.GetWithDefault(IdleSpeed, DefaultKVS)); idleSpeed {
case "", config.EnableOn:
cfg.IdleMode = 0
case config.EnableOff:
cfg.IdleMode = 1
default:
return cfg, fmt.Errorf("unknown value: '%s'", idleSpeed)
}
// Stick to loading deprecated config/env if they are already set, and the Speed value
// has not been changed from its "default" value, if it has been changed honor new settings.
if kvs.GetWithDefault(Speed, DefaultKVS) == "default" {
if kvs.Get(Delay) != "" && kvs.Get(MaxWait) != "" {
if err = lookupDeprecatedScannerConfig(kvs, &cfg); err != nil {
return cfg, err
}
}
}
switch speed := env.Get(EnvSpeed, kvs.GetWithDefault(Speed, DefaultKVS)); speed {
case "fastest":
cfg.Delay, cfg.MaxWait, cfg.Cycle = 0, 0, time.Second
case "fast":
cfg.Delay, cfg.MaxWait, cfg.Cycle = 1, 100*time.Millisecond, time.Minute
case "default":
cfg.Delay, cfg.MaxWait, cfg.Cycle = 2, time.Second, time.Minute
case "slow":
cfg.Delay, cfg.MaxWait, cfg.Cycle = 10, 15*time.Second, time.Minute
case "slowest":
cfg.Delay, cfg.MaxWait, cfg.Cycle = 100, 15*time.Second, 30*time.Minute
default:
return cfg, fmt.Errorf("unknown '%s' value", speed)
}
return cfg, nil
}
func lookupDeprecatedScannerConfig(kvs config.KVS, cfg *Config) (err error) {
delay := env.Get(EnvDelayLegacy, "")
if delay == "" {
delay = env.Get(EnvDelay, kvs.GetWithDefault(Delay, DefaultKVS))
}
cfg.Delay, err = strconv.ParseFloat(delay, 64)
if err != nil {
return err
}
maxWait := env.Get(EnvMaxWaitLegacy, "")
if maxWait == "" {
maxWait = env.Get(EnvMaxWait, kvs.GetWithDefault(MaxWait, DefaultKVS))
}
cfg.MaxWait, err = time.ParseDuration(maxWait)
if err != nil {
return err
}
cycle := env.Get(EnvCycle, kvs.GetWithDefault(Cycle, DefaultKVS))
if cycle == "" {
cycle = "1m"
}
cfg.Cycle, err = time.ParseDuration(cycle)
if err != nil {
return err
}
return nil
}

62
internal/config/server.go Normal file
View File

@@ -0,0 +1,62 @@
// Copyright (c) 2015-2023 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package config
// Opts holds MinIO configuration options
type Opts struct {
FTP struct {
Address string `yaml:"address"`
PassivePortRange string `yaml:"passive-port-range"`
} `yaml:"ftp"`
SFTP struct {
Address string `yaml:"address"`
SSHPrivateKey string `yaml:"ssh-private-key"`
} `yaml:"sftp"`
}
// ServerConfigVersion struct is used to extract the version
type ServerConfigVersion struct {
Version string `yaml:"version"`
}
// ServerConfigCommon struct for server config common options
type ServerConfigCommon struct {
RootUser string `yaml:"rootUser"`
RootPwd string `yaml:"rootPassword"`
Addr string `yaml:"address"`
ConsoleAddr string `yaml:"console-address"`
CertsDir string `yaml:"certs-dir"`
Options Opts `yaml:"options"`
}
// ServerConfigV1 represents a MinIO configuration file v1
type ServerConfigV1 struct {
ServerConfigVersion
ServerConfigCommon
Pools [][]string `yaml:"pools"`
}
// ServerConfig represents a MinIO configuration file
type ServerConfig struct {
ServerConfigVersion
ServerConfigCommon
Pools []struct {
Args []string `yaml:"args"`
SetDriveCount uint64 `yaml:"set-drive-count"`
} `yaml:"pools"`
}

54
internal/config/storageclass/help.go Normal file
View File

@@ -0,0 +1,54 @@
// Copyright (c) 2015-2024 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package storageclass
import "github.com/minio/minio/internal/config"
// Help template for storageclass feature.
var (
defaultHelpPostfix = func(key string) string {
return config.DefaultHelpPostfix(DefaultKVS, key)
}
Help = config.HelpKVS{
config.HelpKV{
Key: ClassStandard,
Description: `set the parity count for default standard storage class` + defaultHelpPostfix(ClassStandard),
Optional: true,
Type: "string",
},
config.HelpKV{
Key: ClassRRS,
Description: `set the parity count for reduced redundancy storage class` + defaultHelpPostfix(ClassRRS),
Optional: true,
Type: "string",
},
config.HelpKV{
Key: Optimize,
Description: `optimize parity calculation for standard storage class, set 'capacity' for capacity optimized (no additional parity)` + defaultHelpPostfix(Optimize),
Optional: true,
Type: "string",
},
config.HelpKV{
Key: config.Comment,
Description: config.DefaultComment,
Optional: true,
Type: "sentence",
},
}
)

40
internal/config/storageclass/legacy.go Normal file
View File

@@ -0,0 +1,40 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package storageclass
import (
"github.com/minio/minio/internal/config"
)
// SetStorageClass - One time migration code needed, for migrating from older config to new for StorageClass.
func SetStorageClass(s config.Config, cfg Config) {
if len(cfg.Standard.String()) == 0 && len(cfg.RRS.String()) == 0 {
// Do not enable storage-class if no settings found.
return
}
s[config.StorageClassSubSys][config.Default] = config.KVS{
config.KV{
Key: ClassStandard,
Value: cfg.Standard.String(),
},
config.KV{
Key: ClassRRS,
Value: cfg.RRS.String(),
},
}
}

435
internal/config/storageclass/storage-class.go Normal file
View File

@@ -0,0 +1,435 @@
// Copyright (c) 2015-2024 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package storageclass
import (
"context"
"encoding/json"
"fmt"
"strconv"
"strings"
"sync"
"github.com/dustin/go-humanize"
"github.com/minio/minio/internal/config"
"github.com/minio/minio/internal/logger"
"github.com/minio/pkg/v3/env"
)
// Standard constants for all storage class
const (
// Reduced redundancy storage class
RRS = "REDUCED_REDUNDANCY"
// Standard storage class
STANDARD = "STANDARD"
)
// Standard constants for config info storage class
const (
ClassStandard = "standard"
ClassRRS = "rrs"
Optimize = "optimize"
InlineBlock = "inline_block"
// Reduced redundancy storage class environment variable
RRSEnv = "MINIO_STORAGE_CLASS_RRS"
// Standard storage class environment variable
StandardEnv = "MINIO_STORAGE_CLASS_STANDARD"
// Optimize storage class environment variable
OptimizeEnv = "MINIO_STORAGE_CLASS_OPTIMIZE"
// Inline block indicates the size of the shard
// that is considered for inlining, remember this
// shard value is the value per drive shard it
// will vary based on the parity that is configured
// for the STANDARD storage_class.
// inlining means data and metadata are written
// together in a single file i.e xl.meta
InlineBlockEnv = "MINIO_STORAGE_CLASS_INLINE_BLOCK"
// Supported storage class scheme is EC
schemePrefix = "EC"
// Min parity drives
minParityDrives = 0
// Default RRS parity is always minimum parity.
defaultRRSParity = 1
)
// DefaultKVS - default storage class config
var (
DefaultKVS = config.KVS{
config.KV{
Key: ClassStandard,
Value: "",
},
config.KV{
Key: ClassRRS,
Value: "EC:1",
},
config.KV{
Key: Optimize,
Value: "availability",
},
config.KV{
Key: InlineBlock,
Value: "",
HiddenIfEmpty: true,
},
}
)
// StorageClass - holds storage class information
type StorageClass struct {
Parity int
}
// ConfigLock is a global lock for storage-class config
var ConfigLock sync.RWMutex
// Config storage class configuration
type Config struct {
Standard StorageClass `json:"standard"`
RRS StorageClass `json:"rrs"`
Optimize string `json:"optimize"`
inlineBlock int64
initialized bool
}
// UnmarshalJSON - Validate SS and RRS parity when unmarshalling JSON.
func (sCfg *Config) UnmarshalJSON(data []byte) error {
type Alias Config
aux := &struct {
*Alias
}{
Alias: (*Alias)(sCfg),
}
return json.Unmarshal(data, &aux)
}
// IsValid - returns true if input string is a valid
// storage class kind supported.
func IsValid(sc string) bool {
return sc == RRS || sc == STANDARD
}
// UnmarshalText unmarshals storage class from its textual form into
// storageClass structure.
func (sc *StorageClass) UnmarshalText(b []byte) error {
scStr := string(b)
if scStr == "" {
return nil
}
s, err := parseStorageClass(scStr)
if err != nil {
return err
}
sc.Parity = s.Parity
return nil
}
// MarshalText - marshals storage class string.
func (sc *StorageClass) MarshalText() ([]byte, error) {
if sc.Parity != 0 {
return fmt.Appendf(nil, "%s:%d", schemePrefix, sc.Parity), nil
}
return []byte{}, nil
}
func (sc *StorageClass) String() string {
if sc.Parity != 0 {
return fmt.Sprintf("%s:%d", schemePrefix, sc.Parity)
}
return ""
}
// Parses given storageClassEnv and returns a storageClass structure.
// Supported Storage Class format is "Scheme:Number of parity drives".
// Currently only supported scheme is "EC".
func parseStorageClass(storageClassEnv string) (sc StorageClass, err error) {
s := strings.Split(storageClassEnv, ":")
// only two elements allowed in the string - "scheme" and "number of parity drives"
if len(s) > 2 {
return StorageClass{}, config.ErrStorageClassValue(nil).Msg("Too many sections in " + storageClassEnv)
} else if len(s) < 2 {
return StorageClass{}, config.ErrStorageClassValue(nil).Msg("Too few sections in " + storageClassEnv)
}
// only allowed scheme is "EC"
if s[0] != schemePrefix {
return StorageClass{}, config.ErrStorageClassValue(nil).Msg("Unsupported scheme " + s[0] + ". Supported scheme is EC")
}
// Number of parity drives should be integer
parityDrives, err := strconv.Atoi(s[1])
if err != nil {
return StorageClass{}, config.ErrStorageClassValue(err)
}
if parityDrives < 0 {
return StorageClass{}, config.ErrStorageClassValue(nil).Msg("Unsupported parity value " + s[1] + " provided")
}
return StorageClass{
Parity: parityDrives,
}, nil
}
// ValidateParity validate standard storage class parity.
func ValidateParity(ssParity, setDriveCount int) error {
// SS parity drives should be greater than or equal to minParityDrives.
// Parity below minParityDrives is not supported.
if ssParity > 0 && ssParity < minParityDrives {
return fmt.Errorf("parity %d should be greater than or equal to %d",
ssParity, minParityDrives)
}
if ssParity > setDriveCount/2 {
return fmt.Errorf("parity %d should be less than or equal to %d", ssParity, setDriveCount/2)
}
return nil
}
// Validates the parity drives.
func validateParity(ssParity, rrsParity, setDriveCount int) (err error) {
// SS parity drives should be greater than or equal to minParityDrives.
// Parity below minParityDrives is not supported.
if ssParity > 0 && ssParity < minParityDrives {
return fmt.Errorf("Standard storage class parity %d should be greater than or equal to %d",
ssParity, minParityDrives)
}
// RRS parity drives should be greater than or equal to minParityDrives.
// Parity below minParityDrives is not supported.
if rrsParity > 0 && rrsParity < minParityDrives {
return fmt.Errorf("Reduced redundancy storage class parity %d should be greater than or equal to %d", rrsParity, minParityDrives)
}
if setDriveCount > 2 {
if ssParity > setDriveCount/2 {
return fmt.Errorf("Standard storage class parity %d should be less than or equal to %d", ssParity, setDriveCount/2)
}
if rrsParity > setDriveCount/2 {
return fmt.Errorf("Reduced redundancy storage class parity %d should be less than or equal to %d", rrsParity, setDriveCount/2)
}
}
if ssParity > 0 && rrsParity > 0 {
if ssParity < rrsParity {
return fmt.Errorf("Standard storage class parity drives %d should be greater than or equal to Reduced redundancy storage class parity drives %d", ssParity, rrsParity)
}
}
return nil
}
// GetParityForSC - Returns the data and parity drive count based on storage class
// If storage class is set using the env vars MINIO_STORAGE_CLASS_RRS and
// MINIO_STORAGE_CLASS_STANDARD or server config fields corresponding values are
// returned.
//
// -- if input storage class is empty then standard is assumed
//
// -- if input is RRS but RRS is not configured/initialized '-1' parity
//
// for RRS is assumed, the caller is expected to choose the right parity
// at that point.
//
// -- if input is STANDARD but STANDARD is not configured/initialized '-1' parity
//
// is returned, the caller is expected to choose the right parity
// at that point.
func (sCfg *Config) GetParityForSC(sc string) (parity int) {
ConfigLock.RLock()
defer ConfigLock.RUnlock()
switch strings.TrimSpace(sc) {
case RRS:
if !sCfg.initialized {
return -1
}
return sCfg.RRS.Parity
default:
if !sCfg.initialized {
return -1
}
return sCfg.Standard.Parity
}
}
// ShouldInline returns true if the shardSize is worthy of inline
// if versioned is true then we chosen 1/8th inline block size
// to satisfy the same constraints.
func (sCfg *Config) ShouldInline(shardSize int64, versioned bool) bool {
if shardSize < 0 {
return false
}
ConfigLock.RLock()
inlineBlock := int64(128 * humanize.KiByte)
if sCfg.initialized {
inlineBlock = sCfg.inlineBlock
}
ConfigLock.RUnlock()
if versioned {
return shardSize <= inlineBlock/8
}
return shardSize <= inlineBlock
}
// InlineBlock indicates the size of the block which will be used to inline
// an erasure shard and written along with xl.meta on the drive, on a versioned
// bucket this value is automatically chosen to 1/8th of the this value, make
// sure to put this into consideration when choosing this value.
func (sCfg *Config) InlineBlock() int64 {
ConfigLock.RLock()
defer ConfigLock.RUnlock()
if !sCfg.initialized {
return 128 * humanize.KiByte
}
return sCfg.inlineBlock
}
// CapacityOptimized - returns true if the storage-class is capacity optimized
// meaning we will not use additional parities when drives are offline.
//
// Default is "availability" optimized, unless this is configured.
func (sCfg *Config) CapacityOptimized() bool {
ConfigLock.RLock()
defer ConfigLock.RUnlock()
if !sCfg.initialized {
return false
}
return sCfg.Optimize == "capacity"
}
// AvailabilityOptimized - returns true if the storage-class is availability
// optimized, meaning we will use additional parities when drives are offline
// to retain parity SLA.
//
// Default is "availability" optimized.
func (sCfg *Config) AvailabilityOptimized() bool {
ConfigLock.RLock()
defer ConfigLock.RUnlock()
if !sCfg.initialized {
return true
}
return sCfg.Optimize == "availability" || sCfg.Optimize == ""
}
// Update update storage-class with new config
func (sCfg *Config) Update(newCfg Config) {
ConfigLock.Lock()
defer ConfigLock.Unlock()
sCfg.RRS = newCfg.RRS
sCfg.Standard = newCfg.Standard
sCfg.Optimize = newCfg.Optimize
sCfg.inlineBlock = newCfg.inlineBlock
sCfg.initialized = true
}
// Enabled returns if storageClass is enabled is enabled.
func Enabled(kvs config.KVS) bool {
ssc := kvs.Get(ClassStandard)
rrsc := kvs.Get(ClassRRS)
return ssc != "" || rrsc != ""
}
// DefaultParityBlocks returns default parity blocks for 'drive' count
func DefaultParityBlocks(drive int) int {
switch drive {
case 1:
return 0
case 3, 2:
return 1
case 4, 5:
return 2
case 6, 7:
return 3
default:
return 4
}
}
// LookupConfig - lookup storage class config and override with valid environment settings if any.
func LookupConfig(kvs config.KVS, setDriveCount int) (cfg Config, err error) {
cfg = Config{}
deprecatedKeys := []string{
"dma",
}
if err = config.CheckValidKeys(config.StorageClassSubSys, kvs, DefaultKVS, deprecatedKeys...); err != nil {
return Config{}, err
}
ssc := env.Get(StandardEnv, kvs.Get(ClassStandard))
rrsc := env.Get(RRSEnv, kvs.Get(ClassRRS))
// Check for environment variables and parse into storageClass struct
if ssc != "" {
cfg.Standard, err = parseStorageClass(ssc)
if err != nil {
return Config{}, err
}
} else {
cfg.Standard.Parity = DefaultParityBlocks(setDriveCount)
}
if rrsc != "" {
cfg.RRS, err = parseStorageClass(rrsc)
if err != nil {
return Config{}, err
}
} else {
cfg.RRS.Parity = defaultRRSParity
if setDriveCount == 1 {
cfg.RRS.Parity = 0
}
}
// Validation is done after parsing both the storage classes. This is needed because we need one
// storage class value to deduce the correct value of the other storage class.
if err = validateParity(cfg.Standard.Parity, cfg.RRS.Parity, setDriveCount); err != nil {
return Config{}, err
}
cfg.Optimize = env.Get(OptimizeEnv, kvs.Get(Optimize))
inlineBlockStr := env.Get(InlineBlockEnv, kvs.Get(InlineBlock))
if inlineBlockStr != "" {
inlineBlock, err := humanize.ParseBytes(inlineBlockStr)
if err != nil {
return cfg, err
}
if inlineBlock > 128*humanize.KiByte {
configLogOnceIf(context.Background(), fmt.Errorf("inline block value bigger than recommended max of 128KiB -> %s, performance may degrade for PUT please benchmark the changes", inlineBlockStr), inlineBlockStr)
}
cfg.inlineBlock = int64(inlineBlock)
} else {
cfg.inlineBlock = 128 * humanize.KiByte
}
cfg.initialized = true
return cfg, nil
}
func configLogOnceIf(ctx context.Context, err error, id string, errKind ...any) {
logger.LogOnceIf(ctx, "config", err, id, errKind...)
}

190
internal/config/storageclass/storage-class_test.go Normal file
View File

@@ -0,0 +1,190 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package storageclass
import (
"errors"
"reflect"
"testing"
)
func TestParseStorageClass(t *testing.T) {
tests := []struct {
storageClassEnv string
wantSc StorageClass
expectedError error
}{
{
"EC:3",
StorageClass{
Parity: 3,
},
nil,
},
{
"EC:4",
StorageClass{
Parity: 4,
},
nil,
},
{
"AB:4",
StorageClass{
Parity: 4,
},
errors.New("Unsupported scheme AB. Supported scheme is EC"),
},
{
"EC:4:5",
StorageClass{
Parity: 4,
},
errors.New("Too many sections in EC:4:5"),
},
{
"EC:A",
StorageClass{
Parity: 4,
},
errors.New(`strconv.Atoi: parsing "A": invalid syntax`),
},
{
"AB",
StorageClass{
Parity: 4,
},
errors.New("Too few sections in AB"),
},
}
for i, tt := range tests {
gotSc, err := parseStorageClass(tt.storageClassEnv)
if err != nil && tt.expectedError == nil {
t.Errorf("Test %d, Expected %s, got %s", i+1, tt.expectedError, err)
return
}
if err == nil && tt.expectedError != nil {
t.Errorf("Test %d, Expected %s, got %s", i+1, tt.expectedError, err)
return
}
if tt.expectedError == nil && !reflect.DeepEqual(gotSc, tt.wantSc) {
t.Errorf("Test %d, Expected %v, got %v", i+1, tt.wantSc, gotSc)
return
}
if tt.expectedError != nil && err.Error() != tt.expectedError.Error() {
t.Errorf("Test %d, Expected `%v`, got `%v`", i+1, tt.expectedError, err)
}
}
}
func TestValidateParity(t *testing.T) {
tests := []struct {
rrsParity int
ssParity int
success bool
setDriveCount int
}{
{2, 4, true, 16},
{3, 3, true, 16},
{0, 0, true, 16},
{1, 4, true, 16},
{0, 4, true, 16},
{7, 6, false, 16},
{9, 0, false, 16},
{9, 9, false, 16},
{2, 9, false, 16},
{9, 2, false, 16},
}
for i, tt := range tests {
err := validateParity(tt.ssParity, tt.rrsParity, tt.setDriveCount)
if err != nil && tt.success {
t.Errorf("Test %d, Expected success, got %s", i+1, err)
}
if err == nil && !tt.success {
t.Errorf("Test %d, Expected failure, got success", i+1)
}
}
}
func TestParityCount(t *testing.T) {
tests := []struct {
sc string
drivesCount int
expectedData int
expectedParity int
}{
{RRS, 16, 14, 2},
{STANDARD, 16, 8, 8},
{"", 16, 8, 8},
{RRS, 16, 9, 7},
{STANDARD, 16, 10, 6},
{"", 16, 9, 7},
}
for i, tt := range tests {
scfg := Config{
Standard: StorageClass{
Parity: 8,
},
RRS: StorageClass{
Parity: 2,
},
initialized: true,
}
// Set env var for test case 4
if i+1 == 4 {
scfg.RRS.Parity = 7
}
// Set env var for test case 5
if i+1 == 5 {
scfg.Standard.Parity = 6
}
// Set env var for test case 6
if i+1 == 6 {
scfg.Standard.Parity = 7
}
parity := scfg.GetParityForSC(tt.sc)
if (tt.drivesCount - parity) != tt.expectedData {
t.Errorf("Test %d, Expected data drives %d, got %d", i+1, tt.expectedData, tt.drivesCount-parity)
continue
}
if parity != tt.expectedParity {
t.Errorf("Test %d, Expected parity drives %d, got %d", i+1, tt.expectedParity, parity)
}
}
}
// Test IsValid method with valid and invalid inputs
func TestIsValidStorageClassKind(t *testing.T) {
tests := []struct {
sc string
want bool
}{
{"STANDARD", true},
{"REDUCED_REDUNDANCY", true},
{"", false},
{"INVALID", false},
{"123", false},
{"MINIO_STORAGE_CLASS_RRS", false},
{"MINIO_STORAGE_CLASS_STANDARD", false},
}
for i, tt := range tests {
if got := IsValid(tt.sc); got != tt.want {
t.Errorf("Test %d, Expected Storage Class to be %t, got %t", i+1, tt.want, got)
}
}
}

154
internal/config/subnet/config.go Normal file
View File

@@ -0,0 +1,154 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package subnet
import (
"net/http"
"net/url"
"os"
"strings"
"sync"
"github.com/minio/minio/internal/config"
"github.com/minio/pkg/v3/env"
xnet "github.com/minio/pkg/v3/net"
)
const (
baseURL = "https://subnet.min.io"
baseURLDev = "http://localhost:9000"
)
// DefaultKVS - default KV config for subnet settings
var DefaultKVS = config.KVS{
config.KV{
Key: config.License, // Deprecated Dec 2021
Value: "",
},
config.KV{
Key: config.APIKey,
Value: "",
},
config.KV{
Key: config.Proxy,
Value: "",
},
}
// Config represents the subnet related configuration
type Config struct {
// The subnet license token - Deprecated Dec 2021
License string `json:"license"`
// The subnet api key
APIKey string `json:"apiKey"`
// The HTTP(S) proxy URL to use for connecting to SUBNET
Proxy string `json:"proxy"`
// Transport configured with proxy_url if set optionally.
transport http.RoundTripper
// The subnet base URL
BaseURL string
}
var configLock sync.RWMutex
// Registered indicates if cluster is registered or not
func (c *Config) Registered() bool {
configLock.RLock()
defer configLock.RUnlock()
return len(c.APIKey) > 0
}
// ApplyEnv - applies the current subnet config to Console UI specific environment variables.
func (c *Config) ApplyEnv() {
configLock.RLock()
defer configLock.RUnlock()
if c.License != "" {
os.Setenv("CONSOLE_SUBNET_LICENSE", c.License)
}
if c.APIKey != "" {
os.Setenv("CONSOLE_SUBNET_API_KEY", c.APIKey)
}
if c.Proxy != "" {
os.Setenv("CONSOLE_SUBNET_PROXY", c.Proxy)
}
os.Setenv("CONSOLE_SUBNET_URL", c.BaseURL)
}
// Update - in-place update with new license and registration information.
func (c *Config) Update(ncfg Config, isDevEnv bool) {
configLock.Lock()
defer configLock.Unlock()
c.License = ncfg.License
c.APIKey = ncfg.APIKey
c.Proxy = ncfg.Proxy
c.transport = ncfg.transport
c.BaseURL = baseURL
if isDevEnv {
c.BaseURL = os.Getenv("_MINIO_SUBNET_URL")
if c.BaseURL == "" {
c.BaseURL = baseURLDev
}
}
}
// LookupConfig - lookup config and override with valid environment settings if any.
func LookupConfig(kvs config.KVS, transport http.RoundTripper) (cfg Config, err error) {
if err = config.CheckValidKeys(config.SubnetSubSys, kvs, DefaultKVS); err != nil {
return cfg, err
}
var proxyURL *xnet.URL
proxy := env.Get(config.EnvMinIOSubnetProxy, kvs.Get(config.Proxy))
if len(proxy) > 0 {
proxyURL, err = xnet.ParseHTTPURL(proxy)
if err != nil {
return cfg, err
}
}
cfg.License = strings.TrimSpace(env.Get(config.EnvMinIOSubnetLicense, kvs.Get(config.License)))
cfg.APIKey = strings.TrimSpace(env.Get(config.EnvMinIOSubnetAPIKey, kvs.Get(config.APIKey)))
cfg.Proxy = proxy
if transport == nil {
// when transport is nil, it means we are just validating the
// inputs not performing any network calls.
return cfg, nil
}
// Make sure to clone the transport before editing the ProxyURL
if proxyURL != nil {
if tr, ok := transport.(*http.Transport); ok {
ctransport := tr.Clone()
ctransport.Proxy = http.ProxyURL((*url.URL)(proxyURL))
cfg.transport = ctransport
}
} else {
cfg.transport = transport
}
return cfg, nil
}

51
internal/config/subnet/help.go Normal file
View File

@@ -0,0 +1,51 @@
// Copyright (c) 2015-2024 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package subnet
import "github.com/minio/minio/internal/config"
var (
defaultHelpPostfix = func(key string) string {
return config.DefaultHelpPostfix(DefaultKVS, key)
}
// HelpSubnet - provides help for subnet api key config
HelpSubnet = config.HelpKVS{
config.HelpKV{
Key: config.License,
Type: "string",
Description: "Enterprise license for the cluster" + defaultHelpPostfix(config.License),
Optional: true,
Sensitive: true,
},
config.HelpKV{
Key: config.APIKey,
Type: "string",
Description: "Enterprise license API key for the cluster" + defaultHelpPostfix(config.APIKey),
Optional: true,
Sensitive: true,
},
config.HelpKV{
Key: config.Proxy,
Type: "string",
Description: "HTTP(s) proxy URL to use for connecting to SUBNET" + defaultHelpPostfix(config.Proxy),
Optional: true,
Sensitive: true,
},
}
)

114
internal/config/subnet/subnet.go Normal file
View File

@@ -0,0 +1,114 @@
// Copyright (c) 2015-2022 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package subnet
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"io"
"mime/multipart"
"net/http"
"time"
xhttp "github.com/minio/minio/internal/http"
)
const (
respBodyLimit = 1 << 20 // 1 MiB
// LoggerWebhookName - subnet logger webhook target
LoggerWebhookName = "subnet"
)
// Upload given file content (payload) to specified URL
func (c Config) Upload(reqURL string, filename string, payload []byte) (string, error) {
if !c.Registered() {
return "", errors.New("Deployment is not registered with SUBNET. Please register the deployment via 'mc license register ALIAS'")
}
var body bytes.Buffer
writer := multipart.NewWriter(&body)
part, e := writer.CreateFormFile("file", filename)
if e != nil {
return "", e
}
if _, e = part.Write(payload); e != nil {
return "", e
}
writer.Close()
r, e := http.NewRequest(http.MethodPost, reqURL, &body)
if e != nil {
return "", e
}
r.Header.Add("Content-Type", writer.FormDataContentType())
return c.submitPost(r)
}
func (c Config) submitPost(r *http.Request) (string, error) {
configLock.RLock()
r.Header.Set(xhttp.SubnetAPIKey, c.APIKey)
configLock.RUnlock()
r.Header.Set(xhttp.MinioDeploymentID, xhttp.GlobalDeploymentID)
client := &http.Client{
Timeout: 10 * time.Second,
Transport: c.transport,
}
resp, err := client.Do(r)
if err != nil {
return "", err
}
defer xhttp.DrainBody(resp.Body)
respBytes, err := io.ReadAll(io.LimitReader(resp.Body, respBodyLimit))
if err != nil {
return "", err
}
respStr := string(respBytes)
if resp.StatusCode == http.StatusOK {
return respStr, nil
}
return respStr, fmt.Errorf("SUBNET request failed with code %d and error: %s", resp.StatusCode, respStr)
}
// Post submit 'payload' to specified URL
func (c Config) Post(reqURL string, payload any) (string, error) {
if !c.Registered() {
return "", errors.New("Deployment is not registered with SUBNET. Please register the deployment via 'mc license register ALIAS'")
}
body, err := json.Marshal(payload)
if err != nil {
return "", err
}
r, err := http.NewRequest(http.MethodPost, reqURL, bytes.NewReader(body))
if err != nil {
return "", err
}
r.Header.Set("Content-Type", "application/json")
return c.submitPost(r)
}