confdroid/confdroid_minio
1
0
Fork 0
Code Issues Packages Wiki Activity

OP#792 initial commit after fork

Browse Source
This commit is contained in:
Arne Teuke
2026-09-27 11:03:48 +02:00
parent 1fc33f4f1f
commit 55b0890196
1338 changed files with 428609 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files

150
internal/logger/audit.go Normal file
View File

@@ -0,0 +1,150 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package logger
import (
"context"
"fmt"
"net/http"
"strconv"
"time"
"github.com/minio/madmin-go/v3/logger/audit"
internalAudit "github.com/minio/minio/internal/logger/message/audit"
"github.com/minio/minio/internal/mcontext"
xhttp "github.com/minio/minio/internal/http"
)
const contextAuditKey = contextKeyType("audit-entry")
// SetAuditEntry sets Audit info in the context.
func SetAuditEntry(ctx context.Context, audit *audit.Entry) context.Context {
if ctx == nil {
LogIf(context.Background(), "audit", fmt.Errorf("context is nil"))
return nil
}
return context.WithValue(ctx, contextAuditKey, audit)
}
// GetAuditEntry returns Audit entry if set.
func GetAuditEntry(ctx context.Context) *audit.Entry {
if ctx != nil {
r, ok := ctx.Value(contextAuditKey).(*audit.Entry)
if ok {
return r
}
r = &audit.Entry{
Version: internalAudit.Version,
DeploymentID: xhttp.GlobalDeploymentID,
Time: time.Now().UTC(),
}
return r
}
return nil
}
// AuditLog - logs audit logs to all audit targets.
func AuditLog(ctx context.Context, w http.ResponseWriter, r *http.Request, reqClaims map[string]any, filterKeys ...string) {
auditTgts := AuditTargets()
if len(auditTgts) == 0 {
return
}
var entry audit.Entry
if w != nil && r != nil {
reqInfo := GetReqInfo(ctx)
if reqInfo == nil {
return
}
reqInfo.RLock()
defer reqInfo.RUnlock()
entry = internalAudit.ToEntry(w, r, reqClaims, xhttp.GlobalDeploymentID)
// indicates all requests for this API call are inbound
entry.Trigger = "incoming"
for _, filterKey := range filterKeys {
delete(entry.ReqClaims, filterKey)
delete(entry.ReqQuery, filterKey)
delete(entry.ReqHeader, filterKey)
delete(entry.RespHeader, filterKey)
}
var (
statusCode int
timeToResponse time.Duration
timeToFirstByte time.Duration
outputBytes int64 = -1 // -1: unknown output bytes
headerBytes int64
)
tc, ok := r.Context().Value(mcontext.ContextTraceKey).(*mcontext.TraceCtxt)
if ok {
statusCode = tc.ResponseRecorder.StatusCode
outputBytes = int64(tc.ResponseRecorder.Size())
headerBytes = int64(tc.ResponseRecorder.HeaderSize())
timeToResponse = time.Now().UTC().Sub(tc.ResponseRecorder.StartTime)
timeToFirstByte = tc.ResponseRecorder.TTFB()
}
entry.AccessKey = reqInfo.Cred.AccessKey
entry.ParentUser = reqInfo.Cred.ParentUser
entry.API.Name = reqInfo.API
entry.API.Bucket = reqInfo.BucketName
entry.API.Object = reqInfo.ObjectName
entry.API.Objects = make([]audit.ObjectVersion, 0, len(reqInfo.Objects))
for _, ov := range reqInfo.Objects {
entry.API.Objects = append(entry.API.Objects, audit.ObjectVersion{
ObjectName: ov.ObjectName,
VersionID: ov.VersionID,
})
}
entry.API.Status = http.StatusText(statusCode)
entry.API.StatusCode = statusCode
entry.API.InputBytes = r.ContentLength
entry.API.OutputBytes = outputBytes
entry.API.HeaderBytes = headerBytes
entry.API.TimeToResponse = strconv.FormatInt(timeToResponse.Nanoseconds(), 10) + "ns"
entry.API.TimeToResponseInNS = strconv.FormatInt(timeToResponse.Nanoseconds(), 10)
// We hold the lock, so we cannot call reqInfo.GetTagsMap().
tags := make(map[string]any, len(reqInfo.tags))
for _, t := range reqInfo.tags {
tags[t.Key] = t.Val
}
entry.Tags = tags
// ignore cases for ttfb when its zero.
if timeToFirstByte != 0 {
entry.API.TimeToFirstByte = strconv.FormatInt(timeToFirstByte.Nanoseconds(), 10) + "ns"
entry.API.TimeToFirstByteInNS = strconv.FormatInt(timeToFirstByte.Nanoseconds(), 10)
}
} else {
auditEntry := GetAuditEntry(ctx)
if auditEntry != nil {
entry = *auditEntry
}
}
// Send audit logs only to http targets.
for _, t := range auditTgts {
if err := t.Send(ctx, entry); err != nil {
LogOnceIf(ctx, "logging", fmt.Errorf("Unable to send audit event(s) to the target `%v`: %v", t, err), "send-audit-event-failure")
}
}
}

657
internal/logger/config.go Normal file
View File

@@ -0,0 +1,657 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package logger
import (
"context"
"crypto/tls"
"errors"
"fmt"
"strconv"
"strings"
"time"
"github.com/minio/pkg/v3/env"
xnet "github.com/minio/pkg/v3/net"
"github.com/minio/minio/internal/config"
"github.com/minio/minio/internal/logger/target/http"
"github.com/minio/minio/internal/logger/target/kafka"
)
// Console logger target
type Console struct {
Enabled bool `json:"enabled"`
}
// Audit/Logger constants
const (
Endpoint = "endpoint"
AuthToken = "auth_token"
ClientCert = "client_cert"
ClientKey = "client_key"
BatchSize = "batch_size"
QueueSize = "queue_size"
QueueDir = "queue_dir"
MaxRetry = "max_retry"
RetryInterval = "retry_interval"
Proxy = "proxy"
httpTimeout = "http_timeout"
KafkaBrokers = "brokers"
KafkaTopic = "topic"
KafkaTLS = "tls"
KafkaTLSSkipVerify = "tls_skip_verify"
KafkaTLSClientAuth = "tls_client_auth"
KafkaSASL = "sasl"
KafkaSASLUsername = "sasl_username"
KafkaSASLPassword = "sasl_password"
KafkaSASLMechanism = "sasl_mechanism"
KafkaClientTLSCert = "client_tls_cert"
KafkaClientTLSKey = "client_tls_key"
KafkaVersion = "version"
KafkaQueueDir = "queue_dir"
KafkaQueueSize = "queue_size"
EnvLoggerWebhookEnable = "MINIO_LOGGER_WEBHOOK_ENABLE"
EnvLoggerWebhookEndpoint = "MINIO_LOGGER_WEBHOOK_ENDPOINT"
EnvLoggerWebhookAuthToken = "MINIO_LOGGER_WEBHOOK_AUTH_TOKEN"
EnvLoggerWebhookClientCert = "MINIO_LOGGER_WEBHOOK_CLIENT_CERT"
EnvLoggerWebhookClientKey = "MINIO_LOGGER_WEBHOOK_CLIENT_KEY"
EnvLoggerWebhookProxy = "MINIO_LOGGER_WEBHOOK_PROXY"
EnvLoggerWebhookBatchSize = "MINIO_LOGGER_WEBHOOK_BATCH_SIZE"
EnvLoggerWebhookQueueSize = "MINIO_LOGGER_WEBHOOK_QUEUE_SIZE"
EnvLoggerWebhookQueueDir = "MINIO_LOGGER_WEBHOOK_QUEUE_DIR"
EnvLoggerWebhookMaxRetry = "MINIO_LOGGER_WEBHOOK_MAX_RETRY"
EnvLoggerWebhookRetryInterval = "MINIO_LOGGER_WEBHOOK_RETRY_INTERVAL"
EnvLoggerWebhookHTTPTimeout = "MINIO_LOGGER_WEBHOOK_HTTP_TIMEOUT"
EnvAuditWebhookEnable = "MINIO_AUDIT_WEBHOOK_ENABLE"
EnvAuditWebhookEndpoint = "MINIO_AUDIT_WEBHOOK_ENDPOINT"
EnvAuditWebhookAuthToken = "MINIO_AUDIT_WEBHOOK_AUTH_TOKEN"
EnvAuditWebhookClientCert = "MINIO_AUDIT_WEBHOOK_CLIENT_CERT"
EnvAuditWebhookClientKey = "MINIO_AUDIT_WEBHOOK_CLIENT_KEY"
EnvAuditWebhookBatchSize = "MINIO_AUDIT_WEBHOOK_BATCH_SIZE"
EnvAuditWebhookQueueSize = "MINIO_AUDIT_WEBHOOK_QUEUE_SIZE"
EnvAuditWebhookQueueDir = "MINIO_AUDIT_WEBHOOK_QUEUE_DIR"
EnvAuditWebhookMaxRetry = "MINIO_AUDIT_WEBHOOK_MAX_RETRY"
EnvAuditWebhookRetryInterval = "MINIO_AUDIT_WEBHOOK_RETRY_INTERVAL"
EnvAuditWebhookHTTPTimeout = "MINIO_AUDIT_WEBHOOK_HTTP_TIMEOUT"
EnvKafkaEnable = "MINIO_AUDIT_KAFKA_ENABLE"
EnvKafkaBrokers = "MINIO_AUDIT_KAFKA_BROKERS"
EnvKafkaTopic = "MINIO_AUDIT_KAFKA_TOPIC"
EnvKafkaTLS = "MINIO_AUDIT_KAFKA_TLS"
EnvKafkaTLSSkipVerify = "MINIO_AUDIT_KAFKA_TLS_SKIP_VERIFY"
EnvKafkaTLSClientAuth = "MINIO_AUDIT_KAFKA_TLS_CLIENT_AUTH"
EnvKafkaSASLEnable = "MINIO_AUDIT_KAFKA_SASL"
EnvKafkaSASLUsername = "MINIO_AUDIT_KAFKA_SASL_USERNAME"
EnvKafkaSASLPassword = "MINIO_AUDIT_KAFKA_SASL_PASSWORD"
EnvKafkaSASLMechanism = "MINIO_AUDIT_KAFKA_SASL_MECHANISM"
EnvKafkaClientTLSCert = "MINIO_AUDIT_KAFKA_CLIENT_TLS_CERT"
EnvKafkaClientTLSKey = "MINIO_AUDIT_KAFKA_CLIENT_TLS_KEY"
EnvKafkaVersion = "MINIO_AUDIT_KAFKA_VERSION"
EnvKafkaQueueDir = "MINIO_AUDIT_KAFKA_QUEUE_DIR"
EnvKafkaQueueSize = "MINIO_AUDIT_KAFKA_QUEUE_SIZE"
loggerTargetNamePrefix = "logger-"
auditTargetNamePrefix = "audit-"
)
var (
errInvalidQueueSize = errors.New("invalid queue_size value")
errInvalidBatchSize = errors.New("invalid batch_size value")
)
// Default KVS for loggerHTTP and loggerAuditHTTP
var (
DefaultLoggerWebhookKVS = config.KVS{
config.KV{
Key: config.Enable,
Value: config.EnableOff,
},
config.KV{
Key: Endpoint,
Value: "",
},
config.KV{
Key: AuthToken,
Value: "",
},
config.KV{
Key: ClientCert,
Value: "",
},
config.KV{
Key: ClientKey,
Value: "",
},
config.KV{
Key: Proxy,
Value: "",
},
config.KV{
Key: BatchSize,
Value: "1",
},
config.KV{
Key: QueueSize,
Value: "100000",
},
config.KV{
Key: QueueDir,
Value: "",
},
config.KV{
Key: MaxRetry,
Value: "0",
},
config.KV{
Key: RetryInterval,
Value: "3s",
},
config.KV{
Key: httpTimeout,
Value: "5s",
},
}
DefaultAuditWebhookKVS = config.KVS{
config.KV{
Key: config.Enable,
Value: config.EnableOff,
},
config.KV{
Key: Endpoint,
Value: "",
},
config.KV{
Key: AuthToken,
Value: "",
},
config.KV{
Key: ClientCert,
Value: "",
},
config.KV{
Key: ClientKey,
Value: "",
},
config.KV{
Key: BatchSize,
Value: "1",
},
config.KV{
Key: QueueSize,
Value: "100000",
},
config.KV{
Key: QueueDir,
Value: "",
},
config.KV{
Key: MaxRetry,
Value: "0",
},
config.KV{
Key: RetryInterval,
Value: "3s",
},
config.KV{
Key: httpTimeout,
Value: "5s",
},
}
DefaultAuditKafkaKVS = config.KVS{
config.KV{
Key: config.Enable,
Value: config.EnableOff,
},
config.KV{
Key: KafkaTopic,
Value: "",
},
config.KV{
Key: KafkaBrokers,
Value: "",
},
config.KV{
Key: KafkaSASLUsername,
Value: "",
},
config.KV{
Key: KafkaSASLPassword,
Value: "",
},
config.KV{
Key: KafkaSASLMechanism,
Value: "plain",
},
config.KV{
Key: KafkaClientTLSCert,
Value: "",
},
config.KV{
Key: KafkaClientTLSKey,
Value: "",
},
config.KV{
Key: KafkaTLSClientAuth,
Value: "0",
},
config.KV{
Key: KafkaSASL,
Value: config.EnableOff,
},
config.KV{
Key: KafkaTLS,
Value: config.EnableOff,
},
config.KV{
Key: KafkaTLSSkipVerify,
Value: config.EnableOff,
},
config.KV{
Key: KafkaVersion,
Value: "",
},
config.KV{
Key: QueueSize,
Value: "100000",
},
config.KV{
Key: QueueDir,
Value: "",
},
}
)
// Config console and http logger targets
type Config struct {
Console Console `json:"console"`
HTTP map[string]http.Config `json:"http"`
AuditWebhook map[string]http.Config `json:"audit"`
AuditKafka map[string]kafka.Config `json:"audit_kafka"`
}
// NewConfig - initialize new logger config.
func NewConfig() Config {
cfg := Config{
// Console logging is on by default
Console: Console{
Enabled: true,
},
HTTP: make(map[string]http.Config),
AuditWebhook: make(map[string]http.Config),
AuditKafka: make(map[string]kafka.Config),
}
return cfg
}
func getCfgVal(envName, key, defaultValue string) string {
if key != config.Default {
envName = envName + config.Default + key
}
return env.Get(envName, defaultValue)
}
func lookupLegacyConfigForSubSys(ctx context.Context, subSys string) Config {
cfg := NewConfig()
switch subSys {
case config.LoggerWebhookSubSys:
var loggerTargets []string
envs := env.List(legacyEnvLoggerHTTPEndpoint)
for _, k := range envs {
target := strings.TrimPrefix(k, legacyEnvLoggerHTTPEndpoint+config.Default)
if target == legacyEnvLoggerHTTPEndpoint {
target = config.Default
}
loggerTargets = append(loggerTargets, target)
}
// Load HTTP logger from the environment if found
for _, target := range loggerTargets {
endpoint := getCfgVal(legacyEnvLoggerHTTPEndpoint, target, "")
if endpoint == "" {
continue
}
url, err := xnet.ParseHTTPURL(endpoint)
if err != nil {
LogOnceIf(ctx, "logging", err, "logger-webhook-"+endpoint)
continue
}
cfg.HTTP[target] = http.Config{
Enabled: true,
Endpoint: url,
}
}
case config.AuditWebhookSubSys:
// List legacy audit ENVs if any.
var loggerAuditTargets []string
envs := env.List(legacyEnvAuditLoggerHTTPEndpoint)
for _, k := range envs {
target := strings.TrimPrefix(k, legacyEnvAuditLoggerHTTPEndpoint+config.Default)
if target == legacyEnvAuditLoggerHTTPEndpoint {
target = config.Default
}
loggerAuditTargets = append(loggerAuditTargets, target)
}
for _, target := range loggerAuditTargets {
endpoint := getCfgVal(legacyEnvAuditLoggerHTTPEndpoint, target, "")
if endpoint == "" {
continue
}
url, err := xnet.ParseHTTPURL(endpoint)
if err != nil {
LogOnceIf(ctx, "logging", err, "audit-webhook-"+endpoint)
continue
}
cfg.AuditWebhook[target] = http.Config{
Enabled: true,
Endpoint: url,
}
}
}
return cfg
}
func lookupAuditKafkaConfig(scfg config.Config, cfg Config) (Config, error) {
for k, kv := range config.Merge(scfg[config.AuditKafkaSubSys], EnvKafkaEnable, DefaultAuditKafkaKVS) {
enabledCfgVal := getCfgVal(EnvKafkaEnable, k, kv.Get(config.Enable))
enabled, err := config.ParseBool(enabledCfgVal)
if err != nil {
return cfg, err
}
if !enabled {
continue
}
var brokers []xnet.Host
kafkaBrokers := getCfgVal(EnvKafkaBrokers, k, kv.Get(KafkaBrokers))
if len(kafkaBrokers) == 0 {
return cfg, config.Errorf("kafka 'brokers' cannot be empty")
}
for s := range strings.SplitSeq(kafkaBrokers, config.ValueSeparator) {
var host *xnet.Host
host, err = xnet.ParseHost(s)
if err != nil {
break
}
brokers = append(brokers, *host)
}
if err != nil {
return cfg, err
}
clientAuthCfgVal := getCfgVal(EnvKafkaTLSClientAuth, k, kv.Get(KafkaTLSClientAuth))
clientAuth, err := strconv.Atoi(clientAuthCfgVal)
if err != nil {
return cfg, err
}
kafkaArgs := kafka.Config{
Enabled: enabled,
Brokers: brokers,
Topic: getCfgVal(EnvKafkaTopic, k, kv.Get(KafkaTopic)),
Version: getCfgVal(EnvKafkaVersion, k, kv.Get(KafkaVersion)),
}
kafkaArgs.TLS.Enable = getCfgVal(EnvKafkaTLS, k, kv.Get(KafkaTLS)) == config.EnableOn
kafkaArgs.TLS.SkipVerify = getCfgVal(EnvKafkaTLSSkipVerify, k, kv.Get(KafkaTLSSkipVerify)) == config.EnableOn
kafkaArgs.TLS.ClientAuth = tls.ClientAuthType(clientAuth)
kafkaArgs.TLS.ClientTLSCert = getCfgVal(EnvKafkaClientTLSCert, k, kv.Get(KafkaClientTLSCert))
kafkaArgs.TLS.ClientTLSKey = getCfgVal(EnvKafkaClientTLSKey, k, kv.Get(KafkaClientTLSKey))
kafkaArgs.SASL.Enable = getCfgVal(EnvKafkaSASLEnable, k, kv.Get(KafkaSASL)) == config.EnableOn
kafkaArgs.SASL.User = getCfgVal(EnvKafkaSASLUsername, k, kv.Get(KafkaSASLUsername))
kafkaArgs.SASL.Password = getCfgVal(EnvKafkaSASLPassword, k, kv.Get(KafkaSASLPassword))
kafkaArgs.SASL.Mechanism = getCfgVal(EnvKafkaSASLMechanism, k, kv.Get(KafkaSASLMechanism))
kafkaArgs.QueueDir = getCfgVal(EnvKafkaQueueDir, k, kv.Get(KafkaQueueDir))
queueSizeCfgVal := getCfgVal(EnvKafkaQueueSize, k, kv.Get(KafkaQueueSize))
queueSize, err := strconv.Atoi(queueSizeCfgVal)
if err != nil {
return cfg, err
}
if queueSize <= 0 {
return cfg, errInvalidQueueSize
}
kafkaArgs.QueueSize = queueSize
cfg.AuditKafka[k] = kafkaArgs
}
return cfg, nil
}
func lookupLoggerWebhookConfig(scfg config.Config, cfg Config) (Config, error) {
for k, kv := range config.Merge(scfg[config.LoggerWebhookSubSys], EnvLoggerWebhookEnable, DefaultLoggerWebhookKVS) {
if v, ok := cfg.HTTP[k]; ok && v.Enabled {
// This target is already enabled using the
// legacy environment variables, ignore.
continue
}
subSysTarget := config.LoggerWebhookSubSys
if k != config.Default {
subSysTarget = config.LoggerWebhookSubSys + config.SubSystemSeparator + k
}
if err := config.CheckValidKeys(subSysTarget, kv, DefaultLoggerWebhookKVS); err != nil {
return cfg, err
}
enableCfgVal := getCfgVal(EnvLoggerWebhookEnable, k, kv.Get(config.Enable))
enable, err := config.ParseBool(enableCfgVal)
if err != nil || !enable {
continue
}
var url *xnet.URL
endpoint := getCfgVal(EnvLoggerWebhookEndpoint, k, kv.Get(Endpoint))
url, err = xnet.ParseHTTPURL(endpoint)
if err != nil {
return cfg, err
}
clientCert := getCfgVal(EnvLoggerWebhookClientCert, k, kv.Get(ClientCert))
clientKey := getCfgVal(EnvLoggerWebhookClientKey, k, kv.Get(ClientKey))
err = config.EnsureCertAndKey(clientCert, clientKey)
if err != nil {
return cfg, err
}
queueSizeCfgVal := getCfgVal(EnvLoggerWebhookQueueSize, k, kv.Get(QueueSize))
queueSize, err := strconv.Atoi(queueSizeCfgVal)
if err != nil {
return cfg, err
}
if queueSize <= 0 {
return cfg, errInvalidQueueSize
}
batchSizeCfgVal := getCfgVal(EnvLoggerWebhookBatchSize, k, kv.Get(BatchSize))
batchSize, err := strconv.Atoi(batchSizeCfgVal)
if err != nil {
return cfg, err
}
if batchSize <= 0 {
return cfg, errInvalidBatchSize
}
maxRetryCfgVal := getCfgVal(EnvLoggerWebhookMaxRetry, k, kv.Get(MaxRetry))
maxRetry, err := strconv.Atoi(maxRetryCfgVal)
if err != nil {
return cfg, err
}
if maxRetry < 0 {
return cfg, fmt.Errorf("invalid %s max_retry", maxRetryCfgVal)
}
retryIntervalCfgVal := getCfgVal(EnvLoggerWebhookRetryInterval, k, kv.Get(RetryInterval))
retryInterval, err := time.ParseDuration(retryIntervalCfgVal)
if err != nil {
return cfg, err
}
if retryInterval > time.Minute {
return cfg, fmt.Errorf("maximum allowed value for retry interval is '1m': %s", retryIntervalCfgVal)
}
httpTimeoutCfgVal := getCfgVal(EnvLoggerWebhookHTTPTimeout, k, kv.Get(httpTimeout))
httpTimeout, err := time.ParseDuration(httpTimeoutCfgVal)
if err != nil {
return cfg, err
}
if httpTimeout < time.Second {
return cfg, fmt.Errorf("minimum value allowed for http_timeout is '1s': %s", httpTimeout)
}
cfg.HTTP[k] = http.Config{
HTTPTimeout: httpTimeout,
Enabled: true,
Endpoint: url,
AuthToken: getCfgVal(EnvLoggerWebhookAuthToken, k, kv.Get(AuthToken)),
ClientCert: clientCert,
ClientKey: clientKey,
Proxy: getCfgVal(EnvLoggerWebhookProxy, k, kv.Get(Proxy)),
BatchSize: batchSize,
QueueSize: queueSize,
QueueDir: getCfgVal(EnvLoggerWebhookQueueDir, k, kv.Get(QueueDir)),
MaxRetry: maxRetry,
RetryIntvl: retryInterval,
Name: loggerTargetNamePrefix + k,
}
}
return cfg, nil
}
func lookupAuditWebhookConfig(scfg config.Config, cfg Config) (Config, error) {
for k, kv := range config.Merge(scfg[config.AuditWebhookSubSys], EnvAuditWebhookEnable, DefaultAuditWebhookKVS) {
if v, ok := cfg.AuditWebhook[k]; ok && v.Enabled {
// This target is already enabled using the
// legacy environment variables, ignore.
continue
}
subSysTarget := config.AuditWebhookSubSys
if k != config.Default {
subSysTarget = config.AuditWebhookSubSys + config.SubSystemSeparator + k
}
if err := config.CheckValidKeys(subSysTarget, kv, DefaultAuditWebhookKVS); err != nil {
return cfg, err
}
enable, err := config.ParseBool(getCfgVal(EnvAuditWebhookEnable, k, kv.Get(config.Enable)))
if err != nil || !enable {
continue
}
var url *xnet.URL
endpoint := getCfgVal(EnvAuditWebhookEndpoint, k, kv.Get(Endpoint))
url, err = xnet.ParseHTTPURL(endpoint)
if err != nil {
return cfg, err
}
clientCert := getCfgVal(EnvAuditWebhookClientCert, k, kv.Get(ClientCert))
clientKey := getCfgVal(EnvAuditWebhookClientKey, k, kv.Get(ClientKey))
err = config.EnsureCertAndKey(clientCert, clientKey)
if err != nil {
return cfg, err
}
queueSizeCfgVal := getCfgVal(EnvAuditWebhookQueueSize, k, kv.Get(QueueSize))
queueSize, err := strconv.Atoi(queueSizeCfgVal)
if err != nil {
return cfg, err
}
if queueSize <= 0 {
return cfg, errInvalidQueueSize
}
batchSizeCfgVal := getCfgVal(EnvAuditWebhookBatchSize, k, kv.Get(BatchSize))
batchSize, err := strconv.Atoi(batchSizeCfgVal)
if err != nil {
return cfg, err
}
if batchSize <= 0 {
return cfg, errInvalidBatchSize
}
maxRetryCfgVal := getCfgVal(EnvAuditWebhookMaxRetry, k, kv.Get(MaxRetry))
maxRetry, err := strconv.Atoi(maxRetryCfgVal)
if err != nil {
return cfg, err
}
if maxRetry < 0 {
return cfg, fmt.Errorf("invalid %s max_retry", maxRetryCfgVal)
}
retryIntervalCfgVal := getCfgVal(EnvAuditWebhookRetryInterval, k, kv.Get(RetryInterval))
retryInterval, err := time.ParseDuration(retryIntervalCfgVal)
if err != nil {
return cfg, err
}
if retryInterval > time.Minute {
return cfg, fmt.Errorf("maximum allowed value for retry interval is '1m': %s", retryIntervalCfgVal)
}
httpTimeoutCfgVal := getCfgVal(EnvAuditWebhookHTTPTimeout, k, kv.Get(httpTimeout))
httpTimeout, err := time.ParseDuration(httpTimeoutCfgVal)
if err != nil {
return cfg, err
}
if httpTimeout < time.Second {
return cfg, fmt.Errorf("minimum value allowed for http_timeout is '1s': %s", httpTimeout)
}
cfg.AuditWebhook[k] = http.Config{
HTTPTimeout: httpTimeout,
Enabled: true,
Endpoint: url,
AuthToken: getCfgVal(EnvAuditWebhookAuthToken, k, kv.Get(AuthToken)),
ClientCert: clientCert,
ClientKey: clientKey,
BatchSize: batchSize,
QueueSize: queueSize,
QueueDir: getCfgVal(EnvAuditWebhookQueueDir, k, kv.Get(QueueDir)),
MaxRetry: maxRetry,
RetryIntvl: retryInterval,
Name: auditTargetNamePrefix + k,
}
}
return cfg, nil
}
// LookupConfigForSubSys - lookup logger config, override with ENVs if set, for the given sub-system
func LookupConfigForSubSys(ctx context.Context, scfg config.Config, subSys string) (cfg Config, err error) {
switch subSys {
case config.LoggerWebhookSubSys:
cfg = lookupLegacyConfigForSubSys(ctx, config.LoggerWebhookSubSys)
if cfg, err = lookupLoggerWebhookConfig(scfg, cfg); err != nil {
return cfg, err
}
case config.AuditWebhookSubSys:
cfg = lookupLegacyConfigForSubSys(ctx, config.AuditWebhookSubSys)
if cfg, err = lookupAuditWebhookConfig(scfg, cfg); err != nil {
return cfg, err
}
case config.AuditKafkaSubSys:
cfg.AuditKafka = make(map[string]kafka.Config)
if cfg, err = lookupAuditKafkaConfig(scfg, cfg); err != nil {
return cfg, err
}
}
return cfg, nil
}
// ValidateSubSysConfig - validates logger related config of given sub-system
func ValidateSubSysConfig(ctx context.Context, scfg config.Config, subSys string) error {
// Lookup for legacy environment variables first
_, err := LookupConfigForSubSys(ctx, scfg, subSys)
return err
}

324
internal/logger/console.go Normal file
View File

@@ -0,0 +1,324 @@
// Copyright (c) 2015-2024 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package logger
import (
"encoding/json"
"fmt"
"os"
"strings"
"time"
"github.com/minio/madmin-go/v3/logger/log"
"github.com/minio/minio/internal/color"
)
// ConsoleLoggerTgt is a stringified value to represent console logging
const ConsoleLoggerTgt = "console+http"
// ExitFunc is called by Fatal() class functions, by default it calls os.Exit()
var ExitFunc = os.Exit
// Logger interface describes the methods that need to be implemented to satisfy the interface requirements.
type Logger interface {
json(msg string, args ...any)
quiet(msg string, args ...any)
pretty(msg string, args ...any)
}
func consoleLog(console Logger, msg string, args ...any) {
switch {
case jsonFlag:
// Strip escape control characters from json message
msg = ansiRE.ReplaceAllLiteralString(msg, "")
console.json(msg, args...)
case quietFlag:
if len(msg) != 0 && len(args) == 0 {
args = append(args, msg)
msg = "%s"
}
console.quiet(msg+"\n", args...)
default:
if len(msg) != 0 && len(args) == 0 {
args = append(args, msg)
msg = "%s"
}
console.pretty(msg+"\n", args...)
}
}
// Fatal prints only fatal error message with no stack trace
// it will be called for input validation failures
func Fatal(err error, msg string, data ...any) {
fatal(err, msg, data...)
}
func fatal(err error, msg string, data ...any) {
if msg == "" {
if len(data) > 0 {
msg = fmt.Sprint(data...)
} else {
msg = "a fatal error"
}
} else {
msg = fmt.Sprintf(msg, data...)
}
consoleLog(fatalMessage, errorFmtFunc(msg, err, jsonFlag))
}
var fatalMessage fatalMsg
type fatalMsg struct{}
func (f fatalMsg) json(msg string, args ...any) {
var message string
if msg != "" {
message = fmt.Sprintf(msg, args...)
} else {
message = fmt.Sprint(args...)
}
logJSON, err := json.Marshal(&log.Entry{
Level: FatalKind,
Message: message,
Time: time.Now().UTC(),
Trace: &log.Trace{Message: message, Source: []string{getSource(6)}},
})
if err != nil {
panic(err)
}
fmt.Fprintln(Output, string(logJSON))
ExitFunc(1)
}
func (f fatalMsg) quiet(msg string, args ...any) {
f.pretty(msg, args...)
}
var (
logTag = "FATAL"
logBanner = color.BgRed(color.FgWhite(color.Bold(logTag))) + " "
emptyBanner = color.BgRed(strings.Repeat(" ", len(logTag))) + " "
bannerWidth = len(logTag) + 1
)
func (f fatalMsg) pretty(msg string, args ...any) {
// Build the passed error message
errMsg := fmt.Sprintf(msg, args...)
tagPrinted := false
// Print the error message: the following code takes care
// of splitting error text and always pretty printing the
// red banner along with the error message. Since the error
// message itself contains some colored text, we needed
// to use some ANSI control escapes to cursor color state
// and freely move in the screen.
for line := range strings.SplitSeq(errMsg, "\n") {
if len(line) == 0 {
// No more text to print, just quit.
break
}
// Save the attributes of the current cursor helps
// us save the text color of the passed error message
ansiSaveAttributes()
// Print banner with or without the log tag
if !tagPrinted {
fmt.Fprint(Output, logBanner)
tagPrinted = true
} else {
fmt.Fprint(Output, emptyBanner)
}
// Restore the text color of the error message
ansiRestoreAttributes()
ansiMoveRight(bannerWidth)
// Continue error message printing
fmt.Fprintln(Output, line)
}
// Exit because this is a fatal error message
ExitFunc(1)
}
type infoMsg struct{}
var info infoMsg
func (i infoMsg) json(msg string, args ...any) {
var message string
if msg != "" {
message = fmt.Sprintf(msg, args...)
} else {
message = fmt.Sprint(args...)
}
logJSON, err := json.Marshal(&log.Entry{
Level: InfoKind,
Message: message,
Time: time.Now().UTC(),
})
if err != nil {
panic(err)
}
fmt.Fprintln(Output, string(logJSON))
}
func (i infoMsg) quiet(msg string, args ...any) {
}
func (i infoMsg) pretty(msg string, args ...any) {
if msg == "" {
fmt.Fprintln(Output, args...)
} else {
fmt.Fprintf(Output, `INFO: `+msg, args...)
}
}
type errorMsg struct{}
var errorMessage errorMsg
func (i errorMsg) json(msg string, args ...any) {
var message string
if msg != "" {
message = fmt.Sprintf(msg, args...)
} else {
message = fmt.Sprint(args...)
}
logJSON, err := json.Marshal(&log.Entry{
Level: ErrorKind,
Message: message,
Time: time.Now().UTC(),
Trace: &log.Trace{Message: message, Source: []string{getSource(6)}},
})
if err != nil {
panic(err)
}
fmt.Fprintln(Output, string(logJSON))
}
func (i errorMsg) quiet(msg string, args ...any) {
i.pretty(msg, args...)
}
func (i errorMsg) pretty(msg string, args ...any) {
if msg == "" {
fmt.Fprintln(Output, args...)
} else {
fmt.Fprintf(Output, `ERRO: `+msg, args...)
}
}
// Error :
func Error(msg string, data ...any) {
if DisableLog {
return
}
consoleLog(errorMessage, msg, data...)
}
// Info :
func Info(msg string, data ...any) {
if DisableLog {
return
}
consoleLog(info, msg, data...)
}
// Startup :
func Startup(msg string, data ...any) {
if DisableLog {
return
}
consoleLog(startup, msg, data...)
}
type startupMsg struct{}
var startup startupMsg
func (i startupMsg) json(msg string, args ...any) {
var message string
if msg != "" {
message = fmt.Sprintf(msg, args...)
} else {
message = fmt.Sprint(args...)
}
logJSON, err := json.Marshal(&log.Entry{
Level: InfoKind,
Message: message,
Time: time.Now().UTC(),
})
if err != nil {
panic(err)
}
fmt.Fprintln(Output, string(logJSON))
}
func (i startupMsg) quiet(msg string, args ...any) {
}
func (i startupMsg) pretty(msg string, args ...any) {
if msg == "" {
fmt.Fprintln(Output, args...)
} else {
fmt.Fprintf(Output, msg, args...)
}
}
type warningMsg struct{}
var warningMessage warningMsg
func (i warningMsg) json(msg string, args ...any) {
var message string
if msg != "" {
message = fmt.Sprintf(msg, args...)
} else {
message = fmt.Sprint(args...)
}
logJSON, err := json.Marshal(&log.Entry{
Level: WarningKind,
Message: message,
Time: time.Now().UTC(),
Trace: &log.Trace{Message: message, Source: []string{getSource(6)}},
})
if err != nil {
panic(err)
}
fmt.Fprintln(Output, string(logJSON))
}
func (i warningMsg) quiet(msg string, args ...any) {
i.pretty(msg, args...)
}
func (i warningMsg) pretty(msg string, args ...any) {
if msg == "" {
fmt.Fprintln(Output, args...)
} else {
fmt.Fprintf(Output, `WARN: `+msg, args...)
}
}
// Warning :
func Warning(msg string, data ...any) {
if DisableLog {
return
}
consoleLog(warningMessage, msg, data...)
}

274
internal/logger/help.go Normal file
View File

@@ -0,0 +1,274 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package logger
import (
"github.com/minio/minio/internal/config"
)
// Help template for logger http and audit
var (
Help = config.HelpKVS{
config.HelpKV{
Key: Endpoint,
Description: `HTTP(s) endpoint e.g. "http://localhost:8080/minio/logs/server"`,
Type: "url",
Sensitive: true,
},
config.HelpKV{
Key: AuthToken,
Description: `opaque string or JWT authorization token`,
Optional: true,
Type: "string",
Sensitive: true,
Secret: true,
},
config.HelpKV{
Key: ClientCert,
Description: "mTLS certificate for webhook authentication",
Optional: true,
Type: "string",
Sensitive: true,
},
config.HelpKV{
Key: ClientKey,
Description: "mTLS certificate key for webhook authentication",
Optional: true,
Type: "string",
Sensitive: true,
},
config.HelpKV{
Key: BatchSize,
Description: "Number of events per HTTP send to webhook target",
Optional: true,
Type: "number",
},
config.HelpKV{
Key: QueueSize,
Description: "configure channel queue size for webhook targets",
Optional: true,
Type: "number",
},
config.HelpKV{
Key: QueueDir,
Description: `staging dir for undelivered logger messages e.g. '/home/logger-events'`,
Optional: true,
Type: "string",
},
config.HelpKV{
Key: Proxy,
Description: "proxy url endpoint e.g. http(s)://proxy",
Optional: true,
Type: "string",
},
config.HelpKV{
Key: MaxRetry,
Description: `maximum retry count before we start dropping logged event(s)`,
Optional: true,
Type: "number",
},
config.HelpKV{
Key: RetryInterval,
Description: `sleep between each retries, allowed maximum value is '1m' e.g. '10s'`,
Optional: true,
Type: "duration",
},
config.HelpKV{
Key: httpTimeout,
Description: `defines the maximum duration for each http request`,
Optional: true,
Type: "duration",
},
config.HelpKV{
Key: config.Comment,
Description: config.DefaultComment,
Optional: true,
Type: "sentence",
},
}
HelpWebhook = config.HelpKVS{
config.HelpKV{
Key: Endpoint,
Description: `HTTP(s) endpoint e.g. "http://localhost:8080/minio/logs/audit"`,
Type: "url",
Sensitive: true,
},
config.HelpKV{
Key: AuthToken,
Description: `opaque string or JWT authorization token`,
Optional: true,
Type: "string",
Sensitive: true,
Secret: true,
},
config.HelpKV{
Key: ClientCert,
Description: "mTLS certificate for webhook authentication",
Optional: true,
Type: "string",
Sensitive: true,
},
config.HelpKV{
Key: ClientKey,
Description: "mTLS certificate key for webhook authentication",
Optional: true,
Type: "string",
Sensitive: true,
},
config.HelpKV{
Key: BatchSize,
Description: "Number of events per HTTP send to webhook target",
Optional: true,
Type: "number",
},
config.HelpKV{
Key: QueueSize,
Description: "configure channel queue size for webhook targets",
Optional: true,
Type: "number",
},
config.HelpKV{
Key: QueueDir,
Description: `staging dir for undelivered audit messages e.g. '/home/audit-events'`,
Optional: true,
Type: "string",
},
config.HelpKV{
Key: MaxRetry,
Description: `maximum retry count before we start dropping audit event(s)`,
Optional: true,
Type: "number",
},
config.HelpKV{
Key: RetryInterval,
Description: `sleep between each retries, allowed maximum value is '1m' e.g. '10s'`,
Optional: true,
Type: "duration",
},
config.HelpKV{
Key: httpTimeout,
Description: `defines the maximum duration for each http request`,
Optional: true,
Type: "duration",
},
config.HelpKV{
Key: config.Comment,
Description: config.DefaultComment,
Optional: true,
Type: "sentence",
},
}
HelpKafka = config.HelpKVS{
config.HelpKV{
Key: KafkaBrokers,
Description: "comma separated list of Kafka broker addresses",
Type: "csv",
},
config.HelpKV{
Key: KafkaTopic,
Description: "Kafka topic used for bucket notifications",
Optional: true,
Type: "string",
},
config.HelpKV{
Key: KafkaSASLUsername,
Description: "username for SASL/PLAIN or SASL/SCRAM authentication",
Optional: true,
Type: "string",
Sensitive: true,
},
config.HelpKV{
Key: KafkaSASLPassword,
Description: "password for SASL/PLAIN or SASL/SCRAM authentication",
Optional: true,
Type: "string",
Sensitive: true,
Secret: true,
},
config.HelpKV{
Key: KafkaSASLMechanism,
Description: "sasl authentication mechanism, default 'plain'",
Optional: true,
Type: "string",
},
config.HelpKV{
Key: KafkaTLSClientAuth,
Description: "clientAuth determines the Kafka server's policy for TLS client auth",
Optional: true,
Type: "string",
},
config.HelpKV{
Key: KafkaSASL,
Description: "set to 'on' to enable SASL authentication",
Optional: true,
Type: "on|off",
},
config.HelpKV{
Key: KafkaTLS,
Description: "set to 'on' to enable TLS",
Optional: true,
Type: "on|off",
},
config.HelpKV{
Key: KafkaTLSSkipVerify,
Description: `trust server TLS without verification, defaults to "on" (verify)`,
Optional: true,
Type: "on|off",
},
config.HelpKV{
Key: KafkaClientTLSCert,
Description: "path to client certificate for mTLS auth",
Optional: true,
Type: "path",
Sensitive: true,
},
config.HelpKV{
Key: KafkaClientTLSKey,
Description: "path to client key for mTLS auth",
Optional: true,
Type: "path",
Sensitive: true,
},
config.HelpKV{
Key: KafkaVersion,
Description: "specify the version of the Kafka cluster",
Optional: true,
Type: "string",
},
config.HelpKV{
Key: QueueSize,
Description: "configure channel queue size for Kafka targets",
Optional: true,
Type: "number",
},
config.HelpKV{
Key: QueueDir,
Description: `staging dir for undelivered audit messages to Kafka e.g. '/home/audit-events'`,
Optional: true,
Type: "string",
},
config.HelpKV{
Key: config.Comment,
Description: config.DefaultComment,
Optional: true,
Type: "sentence",
},
}
)

74
internal/logger/legacy.go Normal file
View File

@@ -0,0 +1,74 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package logger
import (
"github.com/minio/minio/internal/config"
"github.com/minio/minio/internal/logger/target/http"
)
// Legacy envs
const (
legacyEnvAuditLoggerHTTPEndpoint = "MINIO_AUDIT_LOGGER_HTTP_ENDPOINT"
legacyEnvLoggerHTTPEndpoint = "MINIO_LOGGER_HTTP_ENDPOINT"
)
// SetLoggerHTTPAudit - helper for migrating older config to newer KV format.
func SetLoggerHTTPAudit(scfg config.Config, k string, args http.Config) {
if !args.Enabled {
// Do not enable audit targets, if not enabled
return
}
scfg[config.AuditWebhookSubSys][k] = config.KVS{
config.KV{
Key: config.Enable,
Value: config.EnableOn,
},
config.KV{
Key: Endpoint,
Value: args.Endpoint.String(),
},
config.KV{
Key: AuthToken,
Value: args.AuthToken,
},
}
}
// SetLoggerHTTP helper for migrating older config to newer KV format.
func SetLoggerHTTP(scfg config.Config, k string, args http.Config) {
if !args.Enabled {
// Do not enable logger http targets, if not enabled
return
}
scfg[config.LoggerWebhookSubSys][k] = config.KVS{
config.KV{
Key: config.Enable,
Value: config.EnableOn,
},
config.KV{
Key: Endpoint,
Value: args.Endpoint.String(),
},
config.KV{
Key: AuthToken,
Value: args.AuthToken,
},
}
}

460
internal/logger/logger.go Normal file
View File

@@ -0,0 +1,460 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package logger
import (
"context"
"encoding/hex"
"errors"
"fmt"
"go/build"
"io"
"os"
"path/filepath"
"reflect"
"runtime"
"sort"
"strings"
"time"
"github.com/minio/highwayhash"
"github.com/minio/madmin-go/v3"
"github.com/minio/madmin-go/v3/logger/log"
"github.com/minio/minio/internal/color"
xhttp "github.com/minio/minio/internal/http"
)
// HighwayHash key for logging in anonymous mode
var magicHighwayHash256Key = []byte("\x4b\xe7\x34\xfa\x8e\x23\x8a\xcd\x26\x3e\x83\xe6\xbb\x96\x85\x52\x04\x0f\x93\x5d\xa3\x9f\x44\x14\x97\xe0\x9d\x13\x22\xde\x36\xa0")
// Enumerated level types
const (
// Log types errors
FatalKind = madmin.LogKindFatal
WarningKind = madmin.LogKindWarning
ErrorKind = madmin.LogKindError
EventKind = madmin.LogKindEvent
InfoKind = madmin.LogKindInfo
)
var (
// DisableLog avoids printing error/event/info kind of logs
DisableLog = false
// Output allows configuring custom writer, defaults to os.Stderr
Output io.Writer = os.Stderr
)
var trimStrings []string
// TimeFormat - logging time format.
const TimeFormat string = "15:04:05 MST 01/02/2006"
var matchingFuncNames = [...]string{
"http.HandlerFunc.ServeHTTP",
"cmd.serverMain",
// add more here ..
}
// quietFlag: Hide startup messages if enabled
// jsonFlag: Display in JSON format, if enabled
var (
quietFlag, jsonFlag, anonFlag bool
// Custom function to format error
// can be registered by RegisterError
errorFmtFunc = func(introMsg string, err error, jsonFlag bool) string {
return fmt.Sprintf("msg: %s\n err:%s", introMsg, err)
}
)
// EnableQuiet - turns quiet option on.
func EnableQuiet() {
color.TurnOff() // no colored outputs necessary in quiet mode.
quietFlag = true
}
// EnableJSON - outputs logs in json format.
func EnableJSON() {
color.TurnOff() // no colored outputs necessary in JSON mode.
jsonFlag = true
quietFlag = true
}
// EnableAnonymous - turns anonymous flag
// to avoid printing sensitive information.
func EnableAnonymous() {
anonFlag = true
}
// IsJSON - returns true if jsonFlag is true
func IsJSON() bool {
return jsonFlag
}
// IsQuiet - returns true if quietFlag is true
func IsQuiet() bool {
return quietFlag
}
// RegisterError registers the specified rendering function. This latter
// will be called for a pretty rendering of fatal errors.
func RegisterError(f func(string, error, bool) string) {
errorFmtFunc = f
}
// uniq swaps away duplicate elements in data, returning the size of the
// unique set. data is expected to be pre-sorted, and the resulting set in
// the range [0:size] will remain in sorted order. Uniq, following a
// sort.Sort call, can be used to prepare arbitrary inputs for use as sets.
func uniq(data sort.Interface) (size int) {
p, l := 0, data.Len()
if l <= 1 {
return l
}
for i := 1; i < l; i++ {
if !data.Less(p, i) {
continue
}
p++
if p < i {
data.Swap(p, i)
}
}
return p + 1
}
// Remove any duplicates and return unique entries.
func uniqueEntries(paths []string) []string {
sort.Strings(paths)
n := uniq(sort.StringSlice(paths))
return paths[:n]
}
// Init sets the trimStrings to possible GOPATHs
// and GOROOT directories. Also append github.com/minio/minio
// This is done to clean up the filename, when stack trace is
// displayed when an error happens.
func Init(goPath string, goRoot string) {
var goPathList []string
var goRootList []string
var defaultgoPathList []string
var defaultgoRootList []string
pathSeparator := ":"
// Add all possible GOPATH paths into trimStrings
// Split GOPATH depending on the OS type
if runtime.GOOS == "windows" {
pathSeparator = ";"
}
goPathList = strings.Split(goPath, pathSeparator)
goRootList = strings.Split(goRoot, pathSeparator)
defaultgoPathList = strings.Split(build.Default.GOPATH, pathSeparator)
defaultgoRootList = strings.Split(build.Default.GOROOT, pathSeparator)
// Add trim string "{GOROOT}/src/" into trimStrings
trimStrings = []string{filepath.Join(runtime.GOROOT(), "src") + string(filepath.Separator)}
// Add all possible path from GOPATH=path1:path2...:pathN
// as "{path#}/src/" into trimStrings
for _, goPathString := range goPathList {
trimStrings = append(trimStrings, filepath.Join(goPathString, "src")+string(filepath.Separator))
}
for _, goRootString := range goRootList {
trimStrings = append(trimStrings, filepath.Join(goRootString, "src")+string(filepath.Separator))
}
for _, defaultgoPathString := range defaultgoPathList {
trimStrings = append(trimStrings, filepath.Join(defaultgoPathString, "src")+string(filepath.Separator))
}
for _, defaultgoRootString := range defaultgoRootList {
trimStrings = append(trimStrings, filepath.Join(defaultgoRootString, "src")+string(filepath.Separator))
}
// Remove duplicate entries.
trimStrings = uniqueEntries(trimStrings)
// Add "github.com/minio/minio" as the last to cover
// paths like "{GOROOT}/src/github.com/minio/minio"
// and "{GOPATH}/src/github.com/minio/minio"
trimStrings = append(trimStrings, filepath.Join("github.com", "minio", "minio")+string(filepath.Separator))
}
func trimTrace(f string) string {
for _, trimString := range trimStrings {
f = strings.TrimPrefix(filepath.ToSlash(f), filepath.ToSlash(trimString))
}
return filepath.FromSlash(f)
}
func getSource(level int) string {
pc, file, lineNumber, ok := runtime.Caller(level)
if ok {
// Clean up the common prefixes
file = trimTrace(file)
_, funcName := filepath.Split(runtime.FuncForPC(pc).Name())
return fmt.Sprintf("%v:%v:%v()", file, lineNumber, funcName)
}
return ""
}
// getTrace method - creates and returns stack trace
func getTrace(traceLevel int) []string {
var trace []string
pc, file, lineNumber, ok := runtime.Caller(traceLevel)
for ok && file != "" {
// Clean up the common prefixes
file = trimTrace(file)
// Get the function name
_, funcName := filepath.Split(runtime.FuncForPC(pc).Name())
// Skip duplicate traces that start with file name, "<autogenerated>"
// and also skip traces with function name that starts with "runtime."
if !strings.HasPrefix(file, "<autogenerated>") &&
!strings.HasPrefix(funcName, "runtime.") {
// Form and append a line of stack trace into a
// collection, 'trace', to build full stack trace
trace = append(trace, fmt.Sprintf("%v:%v:%v()", file, lineNumber, funcName))
// Ignore trace logs beyond the following conditions
for _, name := range matchingFuncNames {
if funcName == name {
return trace
}
}
}
traceLevel++
// Read stack trace information from PC
pc, file, lineNumber, ok = runtime.Caller(traceLevel)
}
return trace
}
// HashString - return the highway hash of the passed string
func HashString(input string) string {
hh, _ := highwayhash.New(magicHighwayHash256Key)
hh.Write([]byte(input))
return hex.EncodeToString(hh.Sum(nil))
}
// LogAlwaysIf prints a detailed error message during
// the execution of the server.
func LogAlwaysIf(ctx context.Context, subsystem string, err error, errKind ...any) {
if err == nil {
return
}
logIf(ctx, subsystem, err, errKind...)
}
// LogIf prints a detailed error message during
// the execution of the server, if it is not an
// ignored error.
func LogIf(ctx context.Context, subsystem string, err error, errKind ...any) {
if logIgnoreError(err) {
return
}
logIf(ctx, subsystem, err, errKind...)
}
// LogIfNot prints a detailed error message during
// the execution of the server, if it is not an ignored error (either internal or given).
func LogIfNot(ctx context.Context, subsystem string, err error, ignored ...error) {
if logIgnoreError(err) {
return
}
for _, ignore := range ignored {
if errors.Is(err, ignore) {
return
}
}
logIf(ctx, subsystem, err)
}
func errToEntry(ctx context.Context, subsystem string, err error, errKind ...any) log.Entry {
var l string
if anonFlag {
l = reflect.TypeOf(err).String()
} else {
l = fmt.Sprintf("%v (%T)", err, err)
}
return buildLogEntry(ctx, subsystem, l, getTrace(3), errKind...)
}
func logToEntry(ctx context.Context, subsystem, message string, errKind ...any) log.Entry {
return buildLogEntry(ctx, subsystem, message, nil, errKind...)
}
func buildLogEntry(ctx context.Context, subsystem, message string, trace []string, errKind ...any) log.Entry {
logKind := madmin.LogKindError
if len(errKind) > 0 {
if ek, ok := errKind[0].(madmin.LogKind); ok {
logKind = ek
}
}
req := GetReqInfo(ctx)
if req == nil {
req = &ReqInfo{
API: "SYSTEM",
RequestID: fmt.Sprintf("%X", time.Now().UTC().UnixNano()),
}
}
req.RLock()
defer req.RUnlock()
API := "SYSTEM"
switch {
case req.API != "":
API = req.API
case subsystem != "":
API += "." + subsystem
}
// Copy tags. We hold read lock already.
tags := make(map[string]any, len(req.tags))
for _, entry := range req.tags {
tags[entry.Key] = entry.Val
}
// Get the cause for the Error
deploymentID := req.DeploymentID
if req.DeploymentID == "" {
deploymentID = xhttp.GlobalDeploymentID
}
objects := make([]log.ObjectVersion, 0, len(req.Objects))
for _, ov := range req.Objects {
objects = append(objects, log.ObjectVersion{
ObjectName: ov.ObjectName,
VersionID: ov.VersionID,
})
}
entry := log.Entry{
DeploymentID: deploymentID,
Level: logKind,
RemoteHost: req.RemoteHost,
Host: req.Host,
RequestID: req.RequestID,
UserAgent: req.UserAgent,
Time: time.Now().UTC(),
API: &log.API{
Name: API,
Args: &log.Args{
Bucket: req.BucketName,
Object: req.ObjectName,
VersionID: req.VersionID,
Objects: objects,
},
},
}
if trace != nil {
entry.Trace = &log.Trace{
Message: message,
Source: trace,
Variables: tags,
}
} else {
entry.Message = message
}
if anonFlag {
entry.API.Args.Bucket = HashString(entry.API.Args.Bucket)
entry.API.Args.Object = HashString(entry.API.Args.Object)
entry.RemoteHost = HashString(entry.RemoteHost)
if entry.Trace != nil {
entry.Trace.Variables = make(map[string]any)
}
}
return entry
}
// consoleLogIf prints a detailed error message during
// the execution of the server.
func consoleLogIf(ctx context.Context, subsystem string, err error, errKind ...any) {
if DisableLog {
return
}
if err == nil {
return
}
if consoleTgt != nil {
entry := errToEntry(ctx, subsystem, err, errKind...)
consoleTgt.Send(ctx, entry)
}
}
// logIf prints a detailed error message during
// the execution of the server.
func logIf(ctx context.Context, subsystem string, err error, errKind ...any) {
if DisableLog {
return
}
if err == nil {
return
}
entry := errToEntry(ctx, subsystem, err, errKind...)
sendLog(ctx, entry)
}
func sendLog(ctx context.Context, entry log.Entry) {
systemTgts := SystemTargets()
if len(systemTgts) == 0 {
return
}
// Iterate over all logger targets to send the log entry
for _, t := range systemTgts {
if err := t.Send(ctx, entry); err != nil {
if consoleTgt != nil { // Sending to the console never fails
consoleTgt.Send(ctx, errToEntry(ctx, "logging", fmt.Errorf("unable to send log event to Logger target (%s): %v", t.String(), err), entry.Level))
}
}
}
}
// Event sends a event log to log targets
func Event(ctx context.Context, subsystem, msg string, args ...any) {
if DisableLog {
return
}
entry := logToEntry(ctx, subsystem, fmt.Sprintf(msg, args...), EventKind)
sendLog(ctx, entry)
}
// ErrCritical is the value panic'd whenever CriticalIf is called.
var ErrCritical struct{}
// CriticalIf logs the provided error on the console. It fails the
// current go-routine by causing a `panic(ErrCritical)`.
func CriticalIf(ctx context.Context, err error, errKind ...any) {
if err != nil {
LogIf(ctx, "", err, errKind...)
panic(ErrCritical)
}
}
// FatalIf is similar to Fatal() but it ignores passed nil error
func FatalIf(err error, msg string, data ...any) {
if err == nil {
return
}
fatal(err, msg, data...)
}

158
internal/logger/logonce.go Normal file
View File

@@ -0,0 +1,158 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package logger
import (
"context"
"errors"
"sync"
"time"
)
// LogOnce provides the function type for logger.LogOnceIf() function
type LogOnce func(ctx context.Context, err error, id string, errKind ...any)
type onceErr struct {
Err error
Count int
}
// Holds a map of recently logged errors.
type logOnceType struct {
IDMap map[string]onceErr
sync.Mutex
}
func (l *logOnceType) logOnceConsoleIf(ctx context.Context, subsystem string, err error, id string, errKind ...any) {
if err == nil {
return
}
nerr := unwrapErrs(err)
l.Lock()
shouldLog := true
prev, ok := l.IDMap[id]
if !ok {
l.IDMap[id] = onceErr{
Err: nerr,
Count: 1,
}
} else if prev.Err.Error() == nerr.Error() {
// if errors are equal do not log.
prev.Count++
l.IDMap[id] = prev
shouldLog = false
}
l.Unlock()
if shouldLog {
consoleLogIf(ctx, subsystem, err, errKind...)
}
}
const unwrapErrsDepth = 3
// unwrapErrs upto the point where errors.Unwrap(err) returns nil
func unwrapErrs(err error) (leafErr error) {
uerr := errors.Unwrap(err)
depth := 1
for uerr != nil {
// Save the current `uerr`
leafErr = uerr
// continue to look for leaf errors underneath
uerr = errors.Unwrap(leafErr)
depth++
if depth == unwrapErrsDepth {
// If we have reached enough depth we
// do not further recurse down, this
// is done to avoid any unnecessary
// latencies this might bring.
break
}
}
if uerr == nil {
leafErr = err
}
return leafErr
}
// One log message per error.
func (l *logOnceType) logOnceIf(ctx context.Context, subsystem string, err error, id string, errKind ...any) {
if err == nil {
return
}
nerr := unwrapErrs(err)
l.Lock()
shouldLog := true
prev, ok := l.IDMap[id]
if !ok {
l.IDMap[id] = onceErr{
Err: nerr,
Count: 1,
}
} else if prev.Err.Error() == nerr.Error() {
// if errors are equal do not log.
prev.Count++
l.IDMap[id] = prev
shouldLog = false
}
l.Unlock()
if shouldLog {
logIf(ctx, subsystem, err, errKind...)
}
}
// Cleanup the map every one hour so that the log message is printed again for the user to notice.
func (l *logOnceType) cleanupRoutine() {
for {
time.Sleep(time.Hour)
l.Lock()
l.IDMap = make(map[string]onceErr)
l.Unlock()
}
}
// Returns logOnceType
func newLogOnceType() *logOnceType {
l := &logOnceType{IDMap: make(map[string]onceErr)}
go l.cleanupRoutine()
return l
}
var logOnce = newLogOnceType()
// LogOnceIf - Logs notification errors - once per error.
// id is a unique identifier for related log messages, refer to cmd/notification.go
// on how it is used.
func LogOnceIf(ctx context.Context, subsystem string, err error, id string, errKind ...any) {
if logIgnoreError(err) {
return
}
logOnce.logOnceIf(ctx, subsystem, err, id, errKind...)
}
// LogOnceConsoleIf - similar to LogOnceIf but exclusively only logs to console target.
func LogOnceConsoleIf(ctx context.Context, subsystem string, err error, id string, errKind ...any) {
if logIgnoreError(err) {
return
}
logOnce.logOnceConsoleIf(ctx, subsystem, err, id, errKind...)
}

239
internal/logger/logrotate.go Normal file
View File

@@ -0,0 +1,239 @@
// Copyright (c) 2015-2024 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package logger
import (
"encoding/json"
"fmt"
"io"
"os"
"path/filepath"
"time"
"github.com/klauspost/compress/gzip"
"github.com/minio/madmin-go/v3/logger/log"
xioutil "github.com/minio/minio/internal/ioutil"
)
func defaultFilenameFunc() string {
return fmt.Sprintf("minio-%s.log", fmt.Sprintf("%X", time.Now().UTC().UnixNano()))
}
// Options define configuration options for Writer
type Options struct {
// Directory defines the directory where log files will be written to.
// If the directory does not exist, it will be created.
Directory string
// MaximumFileSize defines the maximum size of each log file in bytes.
MaximumFileSize int64
// FileNameFunc specifies the name a new file will take.
// FileNameFunc must ensure collisions in filenames do not occur.
// Do not rely on timestamps to be unique, high throughput writes
// may fall on the same timestamp.
// Eg.
// 2020-03-28_15-00-945-<random-hash>.log
// When FileNameFunc is not specified, DefaultFilenameFunc will be used.
FileNameFunc func() string
// Compress specify if you want the logs to be compressed after rotation.
Compress bool
}
// Writer is a concurrency-safe writer with file rotation.
type Writer struct {
// opts are the configuration options for this Writer
opts Options
// f is the currently open file used for appends.
// Writes to f are only synchronized once Close() is called,
// or when files are being rotated.
f *os.File
pw *xioutil.PipeWriter
pr *xioutil.PipeReader
}
// Write writes p into the current file, rotating if necessary.
// Write is non-blocking, if the writer's queue is not full.
// Write is blocking otherwise.
func (w *Writer) Write(p []byte) (n int, err error) {
return w.pw.Write(p)
}
// Close closes the writer.
// Any accepted writes will be flushed. Any new writes will be rejected.
// Once Close() exits, files are synchronized to disk.
func (w *Writer) Close() error {
w.pw.CloseWithError(nil)
if w.f != nil {
if err := w.closeCurrentFile(); err != nil {
return err
}
}
return nil
}
var stdErrEnc = json.NewEncoder(os.Stderr)
func (w *Writer) listen() {
for {
var r io.Reader = w.pr
if w.opts.MaximumFileSize > 0 {
r = io.LimitReader(w.pr, w.opts.MaximumFileSize)
}
if _, err := io.Copy(w.f, r); err != nil {
msg := fmt.Sprintf("unable to write to log file %v: %v", w.f.Name(), err)
stdErrEnc.Encode(&log.Entry{
Level: ErrorKind,
Message: msg,
Time: time.Now().UTC(),
Trace: &log.Trace{Message: msg},
})
}
if err := w.rotate(); err != nil {
msg := fmt.Sprintf("unable to rotate log file %v: %v", w.f.Name(), err)
stdErrEnc.Encode(&log.Entry{
Level: ErrorKind,
Message: msg,
Time: time.Now().UTC(),
Trace: &log.Trace{Message: msg},
})
}
}
}
func (w *Writer) closeCurrentFile() error {
if err := w.f.Close(); err != nil {
return fmt.Errorf("unable to close current log file: %w", err)
}
return nil
}
func (w *Writer) compress() error {
if !w.opts.Compress {
return nil
}
oldLgFile := w.f.Name()
r, err := os.Open(oldLgFile)
if err != nil {
return err
}
defer r.Close()
gw, err := os.Create(oldLgFile + ".gz")
if err != nil {
return err
}
defer gw.Close()
var wc io.WriteCloser = gzip.NewWriter(gw)
if _, err = io.Copy(wc, r); err != nil {
return err
}
if err = wc.Close(); err != nil {
return err
}
// Persist to disk any caches.
if err = gw.Sync(); err != nil {
return err
}
// close everything before we delete.
if err = gw.Close(); err != nil {
return err
}
if err = r.Close(); err != nil {
return err
}
// Attempt to remove after all fd's are closed.
return os.Remove(oldLgFile)
}
func (w *Writer) rotate() error {
if w.f != nil {
if err := w.closeCurrentFile(); err != nil {
return err
}
// This function is a no-op if opts.Compress is false
// writes an error in JSON form to stderr, if we cannot
// compress.
if err := w.compress(); err != nil {
msg := fmt.Sprintf("unable to compress log file %v: %v, ignoring and moving on", w.f.Name(), err)
stdErrEnc.Encode(&log.Entry{
Level: ErrorKind,
Message: msg,
Time: time.Now().UTC(),
Trace: &log.Trace{Message: msg},
})
}
}
path := filepath.Join(w.opts.Directory, w.opts.FileNameFunc())
f, err := newFile(path)
if err != nil {
return fmt.Errorf("unable to create new file at %v: %w", path, err)
}
w.f = f
return nil
}
// NewDir creates a new concurrency safe Writer which performs log rotation.
func NewDir(opts Options) (io.WriteCloser, error) {
if err := os.MkdirAll(opts.Directory, os.ModePerm); err != nil {
return nil, fmt.Errorf("directory %v does not exist and could not be created: %w", opts.Directory, err)
}
if opts.FileNameFunc == nil {
opts.FileNameFunc = defaultFilenameFunc
}
pr, pw := xioutil.WaitPipe()
w := &Writer{
opts: opts,
pw: pw,
pr: pr,
}
if w.f == nil {
if err := w.rotate(); err != nil {
return nil, fmt.Errorf("Failed to create log file: %w", err)
}
}
go w.listen()
return w, nil
}
func newFile(path string) (*os.File, error) {
return os.OpenFile(path, os.O_WRONLY|os.O_TRUNC|os.O_CREATE|os.O_SYNC, 0o666)
}

79
internal/logger/message/audit/entry.go Normal file
View File

@@ -0,0 +1,79 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package audit
import (
"net/http"
"strings"
"time"
"github.com/minio/madmin-go/v3/logger/audit"
"github.com/minio/minio/internal/handlers"
xhttp "github.com/minio/minio/internal/http"
)
// Version - represents the current version of audit log structure.
const Version = "1"
// NewEntry - constructs an audit entry object with some fields filled
func NewEntry(deploymentID string) audit.Entry {
return audit.Entry{
Version: Version,
DeploymentID: deploymentID,
Time: time.Now().UTC(),
}
}
// ToEntry - constructs an audit entry from a http request
func ToEntry(w http.ResponseWriter, r *http.Request, reqClaims map[string]any, deploymentID string) audit.Entry {
entry := NewEntry(deploymentID)
entry.RemoteHost = handlers.GetSourceIP(r)
entry.UserAgent = r.UserAgent()
entry.ReqClaims = reqClaims
entry.ReqHost = r.Host
entry.ReqPath = r.URL.Path
q := r.URL.Query()
reqQuery := make(map[string]string, len(q))
for k, v := range q {
reqQuery[k] = strings.Join(v, ",")
}
entry.ReqQuery = reqQuery
reqHeader := make(map[string]string, len(r.Header))
for k, v := range r.Header {
reqHeader[k] = strings.Join(v, ",")
}
entry.ReqHeader = reqHeader
wh := w.Header()
entry.RequestID = wh.Get(xhttp.AmzRequestID)
respHeader := make(map[string]string, len(wh))
for k, v := range wh {
respHeader[k] = strings.Join(v, ",")
}
entry.RespHeader = respHeader
if etag := respHeader[xhttp.ETag]; etag != "" {
respHeader[xhttp.ETag] = strings.Trim(etag, `"`)
}
return entry
}

172
internal/logger/reqinfo.go Normal file
View File

@@ -0,0 +1,172 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package logger
import (
"context"
"fmt"
"sync"
"github.com/minio/minio/internal/auth"
)
// Key used for Get/SetReqInfo
type contextKeyType string
const contextLogKey = contextKeyType("miniolog")
// KeyVal - appended to ReqInfo.Tags
type KeyVal struct {
Key string
Val string
}
// ObjectVersion object version key/versionId
type ObjectVersion struct {
ObjectName string
VersionID string `json:"VersionId,omitempty"`
}
// ReqInfo stores the request info.
// Reading/writing directly to struct requires appropriate R/W lock.
type ReqInfo struct {
RemoteHost string // Client Host/IP
Host string // Node Host/IP
UserAgent string // User Agent
DeploymentID string // x-minio-deployment-id
RequestID string // x-amz-request-id
API string // API name - GetObject PutObject NewMultipartUpload etc.
BucketName string `json:",omitempty"` // Bucket name
ObjectName string `json:",omitempty"` // Object name
VersionID string `json:",omitempty"` // corresponding versionID for the object
Objects []ObjectVersion `json:",omitempty"` // Only set during MultiObject delete handler.
Cred auth.Credentials `json:"-"`
Region string `json:"-"`
Owner bool `json:"-"`
AuthType string `json:"-"`
tags []KeyVal // Any additional info not accommodated by above fields
sync.RWMutex
}
// NewReqInfo :
func NewReqInfo(remoteHost, userAgent, deploymentID, requestID, api, bucket, object string) *ReqInfo {
return &ReqInfo{
RemoteHost: remoteHost,
UserAgent: userAgent,
API: api,
DeploymentID: deploymentID,
RequestID: requestID,
BucketName: bucket,
ObjectName: object,
}
}
// AppendTags - appends key/val to ReqInfo.tags
func (r *ReqInfo) AppendTags(key, val string) *ReqInfo {
if r == nil {
return nil
}
r.Lock()
defer r.Unlock()
r.tags = append(r.tags, KeyVal{key, val})
return r
}
// SetTags - sets key/val to ReqInfo.tags
func (r *ReqInfo) SetTags(key, val string) *ReqInfo {
if r == nil {
return nil
}
r.Lock()
defer r.Unlock()
// Search of tag key already exists in tags
var updated bool
for _, tag := range r.tags {
if tag.Key == key {
tag.Val = val
updated = true
break
}
}
if !updated {
// Append to the end of tags list
r.tags = append(r.tags, KeyVal{key, val})
}
return r
}
// GetTags - returns the user defined tags
func (r *ReqInfo) GetTags() []KeyVal {
if r == nil {
return nil
}
r.RLock()
defer r.RUnlock()
return append(make([]KeyVal, 0, len(r.tags)), r.tags...)
}
// GetTagsMap - returns the user defined tags in a map structure
func (r *ReqInfo) GetTagsMap() map[string]string {
if r == nil {
return nil
}
r.RLock()
defer r.RUnlock()
m := make(map[string]string, len(r.tags))
for _, t := range r.tags {
m[t.Key] = t.Val
}
return m
}
// PopulateTagsMap - returns the user defined tags in a map structure
func (r *ReqInfo) PopulateTagsMap(tagsMap map[string]string) {
if r == nil {
return
}
if tagsMap == nil {
return
}
r.RLock()
defer r.RUnlock()
for _, t := range r.tags {
tagsMap[t.Key] = t.Val
}
}
// SetReqInfo sets ReqInfo in the context.
func SetReqInfo(ctx context.Context, req *ReqInfo) context.Context {
if ctx == nil {
LogIf(context.Background(), "", fmt.Errorf("context is nil"))
return nil
}
return context.WithValue(ctx, contextLogKey, req)
}
// GetReqInfo returns ReqInfo if set.
func GetReqInfo(ctx context.Context) *ReqInfo {
if ctx != nil {
r, ok := ctx.Value(contextLogKey).(*ReqInfo)
if ok {
return r
}
r = &ReqInfo{}
return r
}
return nil
}

160
internal/logger/target/console/console.go Normal file
View File

@@ -0,0 +1,160 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package console
import (
"encoding/json"
"fmt"
"io"
"strconv"
"strings"
"github.com/minio/madmin-go/v3/logger/log"
"github.com/minio/minio/internal/color"
"github.com/minio/minio/internal/logger"
)
// Target implements loggerTarget to send log
// in plain or json format to the standard output.
type Target struct {
output io.Writer
}
// Validate - validate if the tty can be written to
func (c *Target) Validate() error {
return nil
}
// Endpoint returns the backend endpoint
func (c *Target) Endpoint() string {
return ""
}
func (c *Target) String() string {
return "console"
}
// Send log message 'e' to console
func (c *Target) Send(e any) error {
entry, ok := e.(log.Entry)
if !ok {
return fmt.Errorf("Uexpected log entry structure %#v", e)
}
if logger.IsJSON() {
logJSON, err := json.Marshal(&entry)
if err != nil {
return err
}
fmt.Fprintln(c.output, string(logJSON))
return nil
}
if entry.Level == logger.EventKind {
fmt.Fprintln(c.output, entry.Message)
return nil
}
traceLength := len(entry.Trace.Source)
trace := make([]string, traceLength)
// Add a sequence number and formatting for each stack trace
// No formatting is required for the first entry
for i, element := range entry.Trace.Source {
trace[i] = fmt.Sprintf("%8v: %s", traceLength-i, element)
}
tagString := ""
for key, value := range entry.Trace.Variables {
if value != "" {
if tagString != "" {
tagString += ", "
}
tagString += fmt.Sprintf("%s=%#v", key, value)
}
}
var apiString string
if entry.API != nil {
apiString = "API: " + entry.API.Name
if entry.API.Args != nil {
args := ""
if entry.API.Args.Bucket != "" {
args = args + "bucket=" + entry.API.Args.Bucket
}
if entry.API.Args.Object != "" {
args = args + ", object=" + entry.API.Args.Object
}
if entry.API.Args.VersionID != "" {
args = args + ", versionId=" + entry.API.Args.VersionID
}
if len(entry.API.Args.Objects) > 0 {
args = args + ", multiObject=true, numberOfObjects=" + strconv.Itoa(len(entry.API.Args.Objects))
}
if len(args) > 0 {
apiString += "(" + args + ")"
}
}
} else {
apiString = "INTERNAL"
}
timeString := "Time: " + entry.Time.Format(logger.TimeFormat)
var deploymentID string
if entry.DeploymentID != "" {
deploymentID = "\nDeploymentID: " + entry.DeploymentID
}
var requestID string
if entry.RequestID != "" {
requestID = "\nRequestID: " + entry.RequestID
}
var remoteHost string
if entry.RemoteHost != "" {
remoteHost = "\nRemoteHost: " + entry.RemoteHost
}
var host string
if entry.Host != "" {
host = "\nHost: " + entry.Host
}
var userAgent string
if entry.UserAgent != "" {
userAgent = "\nUserAgent: " + entry.UserAgent
}
if len(entry.Trace.Variables) > 0 {
tagString = "\n " + tagString
}
msg := color.RedBold(entry.Trace.Message)
output := fmt.Sprintf("\n%s\n%s%s%s%s%s%s\nError: %s%s\n%s",
apiString, timeString, deploymentID, requestID, remoteHost, host, userAgent,
msg, tagString, strings.Join(trace, "\n"))
fmt.Fprintln(c.output, output)
return nil
}
// New initializes a new logger target
// which prints log directly in the standard
// output.
func New(w io.Writer) *Target {
return &Target{output: w}
}

633
internal/logger/target/http/http.go Normal file
View File

@@ -0,0 +1,633 @@
// Copyright (c) 2015-2024 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package http
import (
"bytes"
"context"
"errors"
"fmt"
"net/http"
"net/url"
"os"
"path/filepath"
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
jsoniter "github.com/json-iterator/go"
xhttp "github.com/minio/minio/internal/http"
xioutil "github.com/minio/minio/internal/ioutil"
types "github.com/minio/minio/internal/logger/target/loggertypes"
"github.com/minio/minio/internal/once"
"github.com/minio/minio/internal/store"
xnet "github.com/minio/pkg/v3/net"
"github.com/valyala/bytebufferpool"
)
const (
// maxWorkers is the maximum number of concurrent http loggers
maxWorkers = 16
// maxWorkers is the maximum number of concurrent batch http loggers
maxWorkersWithBatchEvents = 4
// the suffix for the configured queue dir where the logs will be persisted.
httpLoggerExtension = ".http.log"
)
const (
statusOffline = iota
statusOnline
statusClosed
)
var (
logChBuffers = make(map[string]chan any)
logChLock = sync.Mutex{}
)
// Config http logger target
type Config struct {
Enabled bool `json:"enabled"`
Name string `json:"name"`
UserAgent string `json:"userAgent"`
Endpoint *xnet.URL `json:"endpoint"`
AuthToken string `json:"authToken"`
ClientCert string `json:"clientCert"`
ClientKey string `json:"clientKey"`
BatchSize int `json:"batchSize"`
QueueSize int `json:"queueSize"`
QueueDir string `json:"queueDir"`
MaxRetry int `json:"maxRetry"`
RetryIntvl time.Duration `json:"retryInterval"`
Proxy string `json:"string"`
Transport http.RoundTripper `json:"-"`
HTTPTimeout time.Duration `json:"httpTimeout"`
// Custom logger
LogOnceIf func(ctx context.Context, err error, id string, errKind ...any) `json:"-"`
}
// Target implements logger.Target and sends the json
// format of a log entry to the configured http endpoint.
// An internal buffer of logs is maintained but when the
// buffer is full, new logs are just ignored and an error
// is returned to the caller.
type Target struct {
totalMessages atomic.Int64
failedMessages atomic.Int64
status atomic.Int32
// Worker control
workers atomic.Int64
maxWorkers int64
// workerStartMu sync.Mutex
lastStarted time.Time
wg sync.WaitGroup
// Channel of log entries.
// Reading logCh must hold read lock on logChMu (to avoid read race)
// Sending a value on logCh must hold read lock on logChMu (to avoid closing)
logCh chan any
logChMu sync.RWMutex
// If this webhook is being re-configured we will
// assign the new webhook target to this field.
// The Send() method will then re-direct entries
// to the new target when the current one
// has been set to status "statusClosed".
// Once the glogal target slice has been migrated
// the current target will stop receiving entries.
migrateTarget *Target
// Number of events per HTTP send to webhook target
// this is ideally useful only if your endpoint can
// support reading multiple events on a stream for example
// like : Splunk HTTP Event collector, if you are unsure
// set this to '1'.
batchSize int
payloadType string
// store to persist and replay the logs to the target
// to avoid missing events when the target is down.
store store.Store[any]
storeCtxCancel context.CancelFunc
initQueueOnce once.Init
config Config
client *http.Client
httpTimeout time.Duration
}
// Name returns the name of the target
func (h *Target) Name() string {
return "minio-http-" + h.config.Name
}
// Type - returns type of the target
func (h *Target) Type() types.TargetType {
return types.TargetHTTP
}
// Endpoint returns the backend endpoint
func (h *Target) Endpoint() string {
return h.config.Endpoint.String()
}
func (h *Target) String() string {
return h.config.Name
}
// IsOnline returns true if the target is reachable using a cached value
func (h *Target) IsOnline(ctx context.Context) bool {
return h.status.Load() == statusOnline
}
// Stats returns the target statistics.
func (h *Target) Stats() types.TargetStats {
h.logChMu.RLock()
queueLength := len(h.logCh)
h.logChMu.RUnlock()
stats := types.TargetStats{
TotalMessages: h.totalMessages.Load(),
FailedMessages: h.failedMessages.Load(),
QueueLength: queueLength,
}
return stats
}
// AssignMigrateTarget assigns a target
// which will eventually replace the current target.
func (h *Target) AssignMigrateTarget(migrateTgt *Target) {
h.migrateTarget = migrateTgt
}
// Init validate and initialize the http target
func (h *Target) Init(ctx context.Context) (err error) {
if h.config.QueueDir != "" {
return h.initQueueOnce.DoWithContext(ctx, h.initDiskStore)
}
return h.initQueueOnce.DoWithContext(ctx, h.initMemoryStore)
}
func (h *Target) initDiskStore(ctx context.Context) (err error) {
ctx, cancel := context.WithCancel(ctx)
h.storeCtxCancel = cancel
h.lastStarted = time.Now()
go h.startQueueProcessor(ctx, true)
queueStore := store.NewQueueStore[any](
filepath.Join(h.config.QueueDir, h.Name()),
uint64(h.config.QueueSize),
httpLoggerExtension,
)
if err := queueStore.Open(); err != nil {
return fmt.Errorf("unable to initialize the queue store of %s webhook: %w", h.Name(), err)
}
h.store = queueStore
store.StreamItems(h.store, h, ctx.Done(), h.config.LogOnceIf)
return nil
}
func (h *Target) initMemoryStore(ctx context.Context) (err error) {
ctx, cancel := context.WithCancel(ctx)
h.storeCtxCancel = cancel
h.lastStarted = time.Now()
go h.startQueueProcessor(ctx, true)
return nil
}
func (h *Target) send(ctx context.Context, payload []byte, payloadCount int, payloadType string, timeout time.Duration) (err error) {
defer func() {
if err != nil {
if xnet.IsNetworkOrHostDown(err, false) {
h.status.Store(statusOffline)
}
h.failedMessages.Add(int64(payloadCount))
} else {
h.status.Store(statusOnline)
}
}()
ctx, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
req, err := http.NewRequestWithContext(ctx, http.MethodPost,
h.Endpoint(), bytes.NewReader(payload))
if err != nil {
return fmt.Errorf("invalid configuration for '%s'; %v", h.Endpoint(), err)
}
if payloadType != "" {
req.Header.Set(xhttp.ContentType, payloadType)
}
req.Header.Set(xhttp.WebhookEventPayloadCount, strconv.Itoa(payloadCount))
req.Header.Set(xhttp.MinIOVersion, xhttp.GlobalMinIOVersion)
req.Header.Set(xhttp.MinioDeploymentID, xhttp.GlobalDeploymentID)
// Set user-agent to indicate MinIO release
// version to the configured log endpoint
req.Header.Set("User-Agent", h.config.UserAgent)
if h.config.AuthToken != "" {
req.Header.Set("Authorization", h.config.AuthToken)
}
resp, err := h.client.Do(req)
if err != nil {
return fmt.Errorf("%s returned '%w', please check your endpoint configuration", h.Endpoint(), err)
}
// Drain any response.
xhttp.DrainBody(resp.Body)
if resp.StatusCode >= 200 && resp.StatusCode <= 299 {
// accepted HTTP status codes.
return nil
} else if resp.StatusCode == http.StatusForbidden {
return fmt.Errorf("%s returned '%s', please check if your auth token is correctly set", h.Endpoint(), resp.Status)
}
return fmt.Errorf("%s returned '%s', please check your endpoint configuration", h.Endpoint(), resp.Status)
}
func (h *Target) startQueueProcessor(ctx context.Context, mainWorker bool) {
h.logChMu.RLock()
if h.logCh == nil {
h.logChMu.RUnlock()
return
}
h.logChMu.RUnlock()
h.workers.Add(1)
defer h.workers.Add(-1)
h.wg.Add(1)
defer h.wg.Done()
entries := make([]any, 0)
name := h.Name()
defer func() {
// re-load the global buffer pointer
// in case it was modified by a new target.
logChLock.Lock()
currentGlobalBuffer, ok := logChBuffers[name]
logChLock.Unlock()
if !ok {
return
}
for _, v := range entries {
select {
case currentGlobalBuffer <- v:
default:
}
}
if mainWorker {
drain:
for {
select {
case v, ok := <-h.logCh:
if !ok {
break drain
}
currentGlobalBuffer <- v
default:
break drain
}
}
}
}()
lastBatchProcess := time.Now()
buf := bytebufferpool.Get()
enc := jsoniter.ConfigCompatibleWithStandardLibrary.NewEncoder(buf)
defer bytebufferpool.Put(buf)
isDirQueue := h.config.QueueDir != ""
// globalBuffer is always created or adjusted
// before this method is launched.
logChLock.Lock()
globalBuffer := logChBuffers[name]
logChLock.Unlock()
ticker := time.NewTicker(time.Second)
defer ticker.Stop()
var count int
for {
var (
ok bool
entry any
)
if count < h.batchSize {
tickered := false
select {
case <-ticker.C:
tickered = true
case entry = <-globalBuffer:
case entry, ok = <-h.logCh:
if !ok {
return
}
case <-ctx.Done():
return
}
if !tickered {
h.totalMessages.Add(1)
if !isDirQueue {
if err := enc.Encode(&entry); err != nil {
h.config.LogOnceIf(
ctx,
fmt.Errorf("unable to encode webhook log entry, err '%w' entry: %v\n", err, entry),
h.Name(),
)
h.failedMessages.Add(1)
continue
}
} else {
entries = append(entries, entry)
}
count++
}
if len(h.logCh) > 0 || len(globalBuffer) > 0 || count == 0 {
// there is something in the log queue
// process it first, even if we tickered
// first, or we have not received any events
// yet, still wait on it.
continue
}
// If we are doing batching, we should wait
// at least for a second, before sending.
// Even if there is nothing in the queue.
if h.batchSize > 1 && time.Since(lastBatchProcess) < time.Second {
continue
}
}
// if we have reached the count send at once
// or we have crossed last second before batch was sent, send at once
lastBatchProcess = time.Now()
var retries int
retryIntvl := h.config.RetryIntvl
if retryIntvl <= 0 {
retryIntvl = 3 * time.Second
}
maxRetries := h.config.MaxRetry
retry:
// If the channel reaches above half capacity
// we spawn more workers. The workers spawned
// from this main worker routine will exit
// once the channel drops below half capacity
// and when it's been at least 30 seconds since
// we launched a new worker.
if mainWorker && len(h.logCh) > cap(h.logCh)/2 {
nWorkers := h.workers.Load()
if nWorkers < h.maxWorkers {
if time.Since(h.lastStarted).Milliseconds() > 10 {
h.lastStarted = time.Now()
go h.startQueueProcessor(ctx, false)
}
}
}
var err error
if !isDirQueue {
err = h.send(ctx, buf.Bytes(), count, h.payloadType, h.httpTimeout)
} else {
_, err = h.store.PutMultiple(entries)
}
if err != nil {
if errors.Is(err, context.Canceled) {
return
}
h.config.LogOnceIf(
context.Background(),
fmt.Errorf("unable to send audit/log entry(s) to '%s' err '%w': %d", name, err, count),
name,
)
time.Sleep(retryIntvl)
if maxRetries == 0 {
goto retry
}
retries++
if retries <= maxRetries {
goto retry
}
}
entries = make([]any, 0)
count = 0
if !isDirQueue {
buf.Reset()
}
if !mainWorker && len(h.logCh) < cap(h.logCh)/2 {
if time.Since(h.lastStarted).Seconds() > 30 {
return
}
}
}
}
// CreateOrAdjustGlobalBuffer will create or adjust the global log entry buffers
// which are used to migrate log entries between old and new targets.
func CreateOrAdjustGlobalBuffer(currentTgt *Target, newTgt *Target) {
logChLock.Lock()
defer logChLock.Unlock()
requiredCap := currentTgt.config.QueueSize + (currentTgt.config.BatchSize * int(currentTgt.maxWorkers))
currentCap := 0
name := newTgt.Name()
currentBuff, ok := logChBuffers[name]
if !ok {
logChBuffers[name] = make(chan any, requiredCap)
currentCap = requiredCap
} else {
currentCap = cap(currentBuff)
requiredCap += len(currentBuff)
}
if requiredCap > currentCap {
logChBuffers[name] = make(chan any, requiredCap)
if len(currentBuff) > 0 {
drain:
for {
select {
case v, ok := <-currentBuff:
if !ok {
break drain
}
logChBuffers[newTgt.Name()] <- v
default:
break drain
}
}
}
}
}
// New initializes a new logger target which
// sends log over http to the specified endpoint
func New(config Config) (*Target, error) {
maxWorkers := maxWorkers
if config.BatchSize > 100 {
maxWorkers = maxWorkersWithBatchEvents
} else if config.BatchSize <= 0 {
config.BatchSize = 1
}
h := &Target{
logCh: make(chan any, config.QueueSize),
config: config,
batchSize: config.BatchSize,
maxWorkers: int64(maxWorkers),
httpTimeout: config.HTTPTimeout,
}
h.status.Store(statusOffline)
if config.BatchSize > 1 {
h.payloadType = ""
} else {
h.payloadType = "application/json"
}
// If proxy available, set the same
if h.config.Proxy != "" {
proxyURL, _ := url.Parse(h.config.Proxy)
transport := h.config.Transport
if tr, ok := transport.(*http.Transport); ok {
ctransport := tr.Clone()
ctransport.Proxy = http.ProxyURL(proxyURL)
h.config.Transport = ctransport
}
}
h.client = &http.Client{Transport: h.config.Transport}
return h, nil
}
// SendFromStore - reads the log from store and sends it to webhook.
func (h *Target) SendFromStore(key store.Key) (err error) {
var eventData []byte
eventData, err = h.store.GetRaw(key)
if err != nil {
if os.IsNotExist(err) {
return nil
}
return err
}
count := 1
v := strings.Split(key.Name, ":")
if len(v) == 2 {
count, err = strconv.Atoi(v[0])
if err != nil {
return err
}
}
if err := h.send(context.Background(), eventData, count, h.payloadType, h.httpTimeout); err != nil {
return err
}
// Delete the event from store.
return h.store.Del(key)
}
// Send the log message 'entry' to the http target.
// Messages are queued in the disk if the store is enabled
// If Cancel has been called the message is ignored.
func (h *Target) Send(ctx context.Context, entry any) error {
if h.status.Load() == statusClosed {
if h.migrateTarget != nil {
return h.migrateTarget.Send(ctx, entry)
}
return nil
}
h.logChMu.RLock()
defer h.logChMu.RUnlock()
if h.logCh == nil {
// We are closing...
return nil
}
select {
case h.logCh <- entry:
h.totalMessages.Add(1)
case <-ctx.Done():
// return error only for context timedout.
if errors.Is(ctx.Err(), context.DeadlineExceeded) {
return ctx.Err()
}
return nil
default:
h.totalMessages.Add(1)
h.failedMessages.Add(1)
return errors.New("log buffer full")
}
return nil
}
// Cancel - cancels the target.
// All queued messages are flushed and the function returns afterwards.
// All messages sent to the target after this function has been called will be dropped.
func (h *Target) Cancel() {
h.status.Store(statusClosed)
h.storeCtxCancel()
// Wait for messages to be sent...
h.wg.Wait()
// Set logch to nil and close it.
// This will block all Send operations,
// and finish the existing ones.
// All future ones will be discarded.
h.logChMu.Lock()
xioutil.SafeClose(h.logCh)
h.logCh = nil
h.logChMu.Unlock()
}

404
internal/logger/target/kafka/kafka.go Normal file
View File

@@ -0,0 +1,404 @@
// Copyright (c) 2015-2023 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package kafka
import (
"context"
"crypto/tls"
"crypto/x509"
"encoding/json"
"errors"
"fmt"
"log"
"os"
"path/filepath"
"sync"
"sync/atomic"
"time"
"github.com/IBM/sarama"
saramatls "github.com/IBM/sarama/tools/tls"
xioutil "github.com/minio/minio/internal/ioutil"
types "github.com/minio/minio/internal/logger/target/loggertypes"
"github.com/minio/minio/internal/once"
"github.com/minio/minio/internal/store"
xnet "github.com/minio/pkg/v3/net"
)
// the suffix for the configured queue dir where the logs will be persisted.
const kafkaLoggerExtension = ".kafka.log"
const (
statusClosed = iota
statusOffline
statusOnline
)
// Config - kafka target arguments.
type Config struct {
Enabled bool `json:"enable"`
Brokers []xnet.Host `json:"brokers"`
Topic string `json:"topic"`
Version string `json:"version"`
TLS struct {
Enable bool `json:"enable"`
RootCAs *x509.CertPool `json:"-"`
SkipVerify bool `json:"skipVerify"`
ClientAuth tls.ClientAuthType `json:"clientAuth"`
ClientTLSCert string `json:"clientTLSCert"`
ClientTLSKey string `json:"clientTLSKey"`
} `json:"tls"`
SASL struct {
Enable bool `json:"enable"`
User string `json:"username"`
Password string `json:"password"`
Mechanism string `json:"mechanism"`
} `json:"sasl"`
// Queue store
QueueSize int `json:"queueSize"`
QueueDir string `json:"queueDir"`
// Custom logger
LogOnce func(ctx context.Context, err error, id string, errKind ...any) `json:"-"`
}
// Target - Kafka target.
type Target struct {
status int32
totalMessages int64
failedMessages int64
wg sync.WaitGroup
// Channel of log entries.
// Reading logCh must hold read lock on logChMu (to avoid read race)
// Sending a value on logCh must hold read lock on logChMu (to avoid closing)
logCh chan any
logChMu sync.RWMutex
// store to persist and replay the logs to the target
// to avoid missing events when the target is down.
store store.Store[any]
storeCtxCancel context.CancelFunc
initKafkaOnce once.Init
initQueueStoreOnce once.Init
client sarama.Client
producer sarama.SyncProducer
kconfig Config
config *sarama.Config
}
func (h *Target) validate() error {
if len(h.kconfig.Brokers) == 0 {
return errors.New("no broker address found")
}
for _, b := range h.kconfig.Brokers {
if _, err := xnet.ParseHost(b.String()); err != nil {
return err
}
}
return nil
}
// Name returns the name of the target
func (h *Target) Name() string {
return "minio-kafka-audit"
}
// Endpoint - return kafka target
func (h *Target) Endpoint() string {
return "kafka"
}
// String - kafka string
func (h *Target) String() string {
return "kafka"
}
// Stats returns the target statistics.
func (h *Target) Stats() types.TargetStats {
h.logChMu.RLock()
queueLength := len(h.logCh)
h.logChMu.RUnlock()
return types.TargetStats{
TotalMessages: atomic.LoadInt64(&h.totalMessages),
FailedMessages: atomic.LoadInt64(&h.failedMessages),
QueueLength: queueLength,
}
}
// Init initialize kafka target
func (h *Target) Init(ctx context.Context) error {
if !h.kconfig.Enabled {
return nil
}
if err := h.validate(); err != nil {
return err
}
if h.kconfig.QueueDir != "" {
if err := h.initQueueStoreOnce.DoWithContext(ctx, h.initQueueStore); err != nil {
return err
}
return h.initKafkaOnce.Do(h.init)
}
if err := h.init(); err != nil {
return err
}
go h.startKafkaLogger()
return nil
}
func (h *Target) initQueueStore(ctx context.Context) (err error) {
queueDir := filepath.Join(h.kconfig.QueueDir, h.Name())
queueStore := store.NewQueueStore[any](queueDir, uint64(h.kconfig.QueueSize), kafkaLoggerExtension)
if err = queueStore.Open(); err != nil {
return fmt.Errorf("unable to initialize the queue store of %s webhook: %w", h.Name(), err)
}
ctx, cancel := context.WithCancel(ctx)
h.store = queueStore
h.storeCtxCancel = cancel
store.StreamItems(h.store, h, ctx.Done(), h.kconfig.LogOnce)
return err
}
func (h *Target) startKafkaLogger() {
h.logChMu.RLock()
logCh := h.logCh
if logCh != nil {
// We are not allowed to add when logCh is nil
h.wg.Add(1)
defer h.wg.Done()
}
h.logChMu.RUnlock()
if logCh == nil {
return
}
// Create a routine which sends json logs received
// from an internal channel.
for entry := range logCh {
h.logEntry(entry)
}
}
func (h *Target) logEntry(entry any) {
atomic.AddInt64(&h.totalMessages, 1)
if err := h.send(entry); err != nil {
atomic.AddInt64(&h.failedMessages, 1)
h.kconfig.LogOnce(context.Background(), err, h.kconfig.Topic)
}
}
func (h *Target) send(entry any) error {
if err := h.initKafkaOnce.Do(h.init); err != nil {
return err
}
logJSON, err := json.Marshal(&entry)
if err != nil {
return err
}
msg := sarama.ProducerMessage{
Topic: h.kconfig.Topic,
Value: sarama.ByteEncoder(logJSON),
}
_, _, err = h.producer.SendMessage(&msg)
if err != nil {
atomic.StoreInt32(&h.status, statusOffline)
} else {
atomic.StoreInt32(&h.status, statusOnline)
}
return err
}
// Init initialize kafka target
func (h *Target) init() error {
if os.Getenv("_MINIO_KAFKA_DEBUG") != "" {
sarama.DebugLogger = log.Default()
}
sconfig := sarama.NewConfig()
if h.kconfig.Version != "" {
kafkaVersion, err := sarama.ParseKafkaVersion(h.kconfig.Version)
if err != nil {
return err
}
sconfig.Version = kafkaVersion
}
sconfig.Net.KeepAlive = 60 * time.Second
sconfig.Net.SASL.User = h.kconfig.SASL.User
sconfig.Net.SASL.Password = h.kconfig.SASL.Password
initScramClient(h.kconfig, sconfig) // initializes configured scram client.
sconfig.Net.SASL.Enable = h.kconfig.SASL.Enable
tlsConfig, err := saramatls.NewConfig(h.kconfig.TLS.ClientTLSCert, h.kconfig.TLS.ClientTLSKey)
if err != nil {
return err
}
sconfig.Net.TLS.Enable = h.kconfig.TLS.Enable
sconfig.Net.TLS.Config = tlsConfig
sconfig.Net.TLS.Config.InsecureSkipVerify = h.kconfig.TLS.SkipVerify
sconfig.Net.TLS.Config.ClientAuth = h.kconfig.TLS.ClientAuth
sconfig.Net.TLS.Config.RootCAs = h.kconfig.TLS.RootCAs
// These settings are needed to ensure that kafka client doesn't hang on brokers
// refer https://github.com/IBM/sarama/issues/765#issuecomment-254333355
sconfig.Producer.Retry.Max = 2
sconfig.Producer.Retry.Backoff = (10 * time.Second)
sconfig.Producer.Return.Successes = true
sconfig.Producer.Return.Errors = true
sconfig.Producer.RequiredAcks = 1
sconfig.Producer.Timeout = (10 * time.Second)
sconfig.Net.ReadTimeout = (10 * time.Second)
sconfig.Net.DialTimeout = (10 * time.Second)
sconfig.Net.WriteTimeout = (10 * time.Second)
sconfig.Metadata.Retry.Max = 1
sconfig.Metadata.Retry.Backoff = (10 * time.Second)
sconfig.Metadata.RefreshFrequency = (15 * time.Minute)
h.config = sconfig
var brokers []string
for _, broker := range h.kconfig.Brokers {
brokers = append(brokers, broker.String())
}
client, err := sarama.NewClient(brokers, sconfig)
if err != nil {
return err
}
producer, err := sarama.NewSyncProducerFromClient(client)
if err != nil {
return err
}
h.client = client
h.producer = producer
if len(h.client.Brokers()) > 0 {
// Refer https://github.com/IBM/sarama/issues/1341
atomic.StoreInt32(&h.status, statusOnline)
}
return nil
}
// IsOnline returns true if the target is online.
func (h *Target) IsOnline(_ context.Context) bool {
return atomic.LoadInt32(&h.status) == statusOnline
}
// Send log message 'e' to kafka target.
func (h *Target) Send(ctx context.Context, entry any) error {
if h.store != nil {
// save the entry to the queue store which will be replayed to the target.
_, err := h.store.Put(entry)
return err
}
h.logChMu.RLock()
defer h.logChMu.RUnlock()
if h.logCh == nil {
// We are closing...
return nil
}
select {
case h.logCh <- entry:
case <-ctx.Done():
// return error only for context timedout.
if errors.Is(ctx.Err(), context.DeadlineExceeded) {
return ctx.Err()
}
return nil
default:
// log channel is full, do not wait and return
// an error immediately to the caller
atomic.AddInt64(&h.totalMessages, 1)
atomic.AddInt64(&h.failedMessages, 1)
return errors.New("log buffer full")
}
return nil
}
// SendFromStore - reads the log from store and sends it to kafka.
func (h *Target) SendFromStore(key store.Key) (err error) {
auditEntry, err := h.store.Get(key)
if err != nil {
if os.IsNotExist(err) {
return nil
}
return err
}
atomic.AddInt64(&h.totalMessages, 1)
err = h.send(auditEntry)
if err != nil {
atomic.AddInt64(&h.failedMessages, 1)
return err
}
// Delete the event from store.
return h.store.Del(key)
}
// Cancel - cancels the target
func (h *Target) Cancel() {
// If queuestore is configured, cancel it's context to
// stop the replay go-routine.
if h.store != nil {
h.storeCtxCancel()
}
// Set logch to nil and close it.
// This will block all Send operations,
// and finish the existing ones.
// All future ones will be discarded.
h.logChMu.Lock()
xioutil.SafeClose(h.logCh)
h.logCh = nil
h.logChMu.Unlock()
if h.producer != nil {
h.producer.Close()
h.client.Close()
}
// Wait for messages to be sent...
h.wg.Wait()
}
// New initializes a new logger target which
// sends log over http to the specified endpoint
func New(config Config) *Target {
target := &Target{
logCh: make(chan any, config.QueueSize),
kconfig: config,
status: statusOffline,
}
return target
}
// Type - returns type of the target
func (h *Target) Type() types.TargetType {
return types.TargetKafka
}

87
internal/logger/target/kafka/kafka_scram_client_contrib.go Normal file
View File

@@ -0,0 +1,87 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package kafka
import (
"crypto/sha512"
"strings"
"github.com/IBM/sarama"
"github.com/xdg/scram"
"github.com/minio/minio/internal/hash/sha256"
)
func initScramClient(cfg Config, config *sarama.Config) {
switch strings.ToLower(cfg.SASL.Mechanism) {
case "sha512":
config.Net.SASL.SCRAMClientGeneratorFunc = func() sarama.SCRAMClient { return &XDGSCRAMClient{HashGeneratorFcn: KafkaSHA512} }
config.Net.SASL.Mechanism = sarama.SASLMechanism(sarama.SASLTypeSCRAMSHA512)
case "sha256":
config.Net.SASL.SCRAMClientGeneratorFunc = func() sarama.SCRAMClient { return &XDGSCRAMClient{HashGeneratorFcn: KafkaSHA256} }
config.Net.SASL.Mechanism = sarama.SASLMechanism(sarama.SASLTypeSCRAMSHA256)
default:
// default to PLAIN
config.Net.SASL.Mechanism = sarama.SASLMechanism(sarama.SASLTypePlaintext)
}
}
// KafkaSHA256 is a function that returns a crypto/sha256 hasher and should be used
// to create Client objects configured for SHA-256 hashing.
var KafkaSHA256 scram.HashGeneratorFcn = sha256.New
// KafkaSHA512 is a function that returns a crypto/sha512 hasher and should be used
// to create Client objects configured for SHA-512 hashing.
var KafkaSHA512 scram.HashGeneratorFcn = sha512.New
// XDGSCRAMClient implements the client-side of an authentication
// conversation with a server. A new conversation must be created for
// each authentication attempt.
type XDGSCRAMClient struct {
*scram.Client
*scram.ClientConversation
scram.HashGeneratorFcn
}
// Begin constructs a SCRAM client component based on a given hash.Hash
// factory receiver. This constructor will normalize the username, password
// and authzID via the SASLprep algorithm, as recommended by RFC-5802. If
// SASLprep fails, the method returns an error.
func (x *XDGSCRAMClient) Begin(userName, password, authzID string) (err error) {
x.Client, err = x.NewClient(userName, password, authzID)
if err != nil {
return err
}
x.ClientConversation = x.NewConversation()
return nil
}
// Step takes a string provided from a server (or just an empty string for the
// very first conversation step) and attempts to move the authentication
// conversation forward. It returns a string to be sent to the server or an
// error if the server message is invalid. Calling Step after a conversation
// completes is also an error.
func (x *XDGSCRAMClient) Step(challenge string) (response string, err error) {
response, err = x.ClientConversation.Step(challenge)
return response, err
}
// Done returns true if the conversation is completed or has errored.
func (x *XDGSCRAMClient) Done() bool {
return x.ClientConversation.Done()
}

26
internal/logger/target/loggertypes/targettype_string.go Normal file
View File

@@ -0,0 +1,26 @@
// Code generated by "stringer -type=TargetType -trimprefix=Target types.go"; DO NOT EDIT.
package loggertypes
import "strconv"
func _() {
// An "invalid array index" compiler error signifies that the constant values have changed.
// Re-run the stringer command to generate them again.
var x [1]struct{}
_ = x[TargetConsole-1]
_ = x[TargetHTTP-2]
_ = x[TargetKafka-3]
}
const _TargetType_name = "ConsoleHTTPKafka"
var _TargetType_index = [...]uint8{0, 7, 11, 16}
func (i TargetType) String() string {
i -= 1
if i >= TargetType(len(_TargetType_index)-1) {
return "TargetType(" + strconv.FormatInt(int64(i+1), 10) + ")"
}
return _TargetType_name[_TargetType_index[i]:_TargetType_index[i+1]]
}

43
internal/logger/target/loggertypes/types.go Normal file
View File

@@ -0,0 +1,43 @@
// Copyright (c) 2015-2022 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package loggertypes
// TargetType indicates type of the target e.g. console, http, kafka
type TargetType uint8
//go:generate stringer -type=TargetType -trimprefix=Target $GOFILE
// Constants for target types
const (
_ TargetType = iota
TargetConsole
TargetHTTP
TargetKafka
)
// TargetStats contains statistics for a target.
type TargetStats struct {
// QueueLength is the queue length if any messages are queued.
QueueLength int
// TotalMessages is the total number of messages sent in the lifetime of the target
TotalMessages int64
// FailedMessages should log message count that failed to send.
FailedMessages int64
}

170
internal/logger/target/testlogger/testlogger.go Normal file
View File

@@ -0,0 +1,170 @@
// Copyright (c) 2015-2023 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
// Package testlogger contains an autoregistering logger that can be used to capture logging events
// for individual tests.
// This package should only be included by test files.
// To enable logging for a test, use:
//
// func TestSomething(t *testing.T) {
// defer testlogger.T.SetLogTB(t)()
//
// This cannot be used for parallel tests.
package testlogger
import (
"context"
"fmt"
"os"
"strings"
"sync/atomic"
"testing"
"github.com/minio/madmin-go/v3/logger/log"
"github.com/minio/minio/internal/logger"
types "github.com/minio/minio/internal/logger/target/loggertypes"
)
const (
logMessage = iota
errorMessage
fatalMessage
)
// T is the test logger.
var T = &testLogger{}
func init() {
logger.AddSystemTarget(context.Background(), T)
}
type testLogger struct {
current atomic.Pointer[testing.TB]
action atomic.Int32
}
// SetLogTB will set the logger to output to tb.
// Call the returned function to disable logging.
func (t *testLogger) SetLogTB(tb testing.TB) func() {
return t.setTB(tb, logMessage)
}
// SetErrorTB will set the logger to output to tb.Error.
// Call the returned function to disable logging.
func (t *testLogger) SetErrorTB(tb testing.TB) func() {
return t.setTB(tb, errorMessage)
}
// SetFatalTB will set the logger to output to tb.Panic.
// Call the returned function to disable logging.
func (t *testLogger) SetFatalTB(tb testing.TB) func() {
return t.setTB(tb, fatalMessage)
}
func (t *testLogger) setTB(tb testing.TB, action int32) func() {
old := t.action.Swap(action)
t.current.Store(&tb)
return func() {
t.current.Store(nil)
t.action.Store(old)
}
}
func (t *testLogger) String() string {
tb := t.current.Load()
if tb != nil {
tbb := *tb
return tbb.Name()
}
return ""
}
func (t *testLogger) Endpoint() string {
return ""
}
func (t *testLogger) Stats() types.TargetStats {
return types.TargetStats{}
}
func (t *testLogger) Init(ctx context.Context) error {
return nil
}
func (t *testLogger) IsOnline(ctx context.Context) bool {
return t.current.Load() != nil
}
func (t *testLogger) Cancel() {
t.current.Store(nil)
}
func (t *testLogger) Send(ctx context.Context, entry any) error {
tb := t.current.Load()
var logf func(format string, args ...any)
if tb != nil {
tbb := *tb
tbb.Helper()
switch t.action.Load() {
case errorMessage:
logf = tbb.Errorf
case fatalMessage:
logf = tbb.Fatalf
default:
logf = tbb.Logf
}
} else {
switch t.action.Load() {
case errorMessage:
logf = func(format string, args ...any) {
fmt.Fprintf(os.Stderr, format+"\n", args...)
}
case fatalMessage:
logf = func(format string, args ...any) {
fmt.Fprintf(os.Stderr, format+"\n", args...)
}
defer os.Exit(1)
default:
logf = func(format string, args ...any) {
fmt.Fprintf(os.Stdout, format+"\n", args...)
}
}
}
switch v := entry.(type) {
case log.Entry:
if v.Trace == nil {
logf("%s: %s", v.Level, v.Message)
} else {
msg := fmt.Sprintf("%s: %+v", v.Level, v.Trace.Message)
for i, m := range v.Trace.Source {
if i == 0 && strings.Contains(m, "logger.go:") {
continue
}
msg += fmt.Sprintf("\n%s", m)
}
logf("%s", msg)
}
default:
logf("%+v (%T)", v, v)
}
return nil
}
func (t *testLogger) Type() types.TargetType {
return types.TargetConsole
}

251
internal/logger/targets.go Normal file
View File

@@ -0,0 +1,251 @@
// Copyright (c) 2015-2022 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package logger
import (
"context"
"fmt"
"strings"
"sync"
"github.com/minio/minio/internal/logger/target/http"
"github.com/minio/minio/internal/logger/target/kafka"
types "github.com/minio/minio/internal/logger/target/loggertypes"
)
// Target is the entity that we will receive
// a single log entry and Send it to the log target
//
// e.g. Send the log to a http server
type Target interface {
String() string
Endpoint() string
Stats() types.TargetStats
Init(ctx context.Context) error
IsOnline(ctx context.Context) bool
Cancel()
Send(ctx context.Context, entry any) error
Type() types.TargetType
}
type targetsList struct {
list []Target
mu sync.RWMutex
}
func newTargetsList() *targetsList {
return &targetsList{}
}
func (tl *targetsList) get() []Target {
tl.mu.RLock()
defer tl.mu.RUnlock()
return tl.list
}
func (tl *targetsList) add(t Target) {
tl.mu.Lock()
defer tl.mu.Unlock()
tl.list = append(tl.list, t)
}
func (tl *targetsList) set(tgts []Target) {
tl.mu.Lock()
defer tl.mu.Unlock()
tl.list = tgts
}
var (
// systemTargets is the set of enabled loggers.
systemTargets = newTargetsList()
// auditTargets is the list of enabled audit loggers
auditTargets = newTargetsList()
// This is always set represent /dev/console target
consoleTgt Target
)
// SystemTargets returns active targets.
// Returned slice may not be modified in any way.
func SystemTargets() []Target {
return systemTargets.get()
}
// AuditTargets returns active audit targets.
// Returned slice may not be modified in any way.
func AuditTargets() []Target {
return auditTargets.get()
}
// CurrentStats returns the current statistics.
func CurrentStats() map[string]types.TargetStats {
sys := SystemTargets()
audit := AuditTargets()
res := make(map[string]types.TargetStats, len(sys)+len(audit))
cnt := make(map[string]int, len(sys)+len(audit))
// Add system and audit.
for _, t := range sys {
key := strings.ToLower(t.Type().String())
n := cnt[key]
cnt[key]++
key = fmt.Sprintf("sys_%s_%d", key, n)
res[key] = t.Stats()
}
for _, t := range audit {
key := strings.ToLower(t.Type().String())
n := cnt[key]
cnt[key]++
key = fmt.Sprintf("audit_%s_%d", key, n)
res[key] = t.Stats()
}
return res
}
// AddSystemTarget adds a new logger target to the
// list of enabled loggers
func AddSystemTarget(ctx context.Context, t Target) error {
if err := t.Init(ctx); err != nil {
return err
}
if consoleTgt == nil {
if t.Type() == types.TargetConsole {
consoleTgt = t
}
}
systemTargets.add(t)
return nil
}
func initKafkaTargets(ctx context.Context, cfgMap map[string]kafka.Config) ([]Target, []error) {
tgts := []Target{}
errs := []error{}
for _, l := range cfgMap {
if l.Enabled {
t := kafka.New(l)
tgts = append(tgts, t)
e := t.Init(ctx)
if e != nil {
errs = append(errs, e)
}
}
}
return tgts, errs
}
// Split targets into two groups:
//
// group1 contains all targets of type t
// group2 contains the remaining targets
func splitTargets(targets []Target, t types.TargetType) (group1 []Target, group2 []Target) {
for _, target := range targets {
if target.Type() == t {
group1 = append(group1, target)
} else {
group2 = append(group2, target)
}
}
return group1, group2
}
func cancelTargets(targets []Target) {
for _, target := range targets {
go target.Cancel()
}
}
// UpdateHTTPWebhooks swaps system webhook targets with newly loaded ones from the cfg
func UpdateHTTPWebhooks(ctx context.Context, cfgs map[string]http.Config) (errs []error) {
return updateHTTPTargets(ctx, cfgs, systemTargets)
}
// UpdateAuditWebhooks swaps audit webhook targets with newly loaded ones from the cfg
func UpdateAuditWebhooks(ctx context.Context, cfgs map[string]http.Config) (errs []error) {
return updateHTTPTargets(ctx, cfgs, auditTargets)
}
func updateHTTPTargets(ctx context.Context, cfgs map[string]http.Config, targetsList *targetsList) (errs []error) {
tgts := make([]*http.Target, 0)
newWebhooks := make([]Target, 0)
for _, cfg := range cfgs {
if cfg.Enabled {
t, err := http.New(cfg)
if err != nil {
errs = append(errs, err)
}
tgts = append(tgts, t)
newWebhooks = append(newWebhooks, t)
}
}
oldTargets, others := splitTargets(targetsList.get(), types.TargetHTTP)
newWebhooks = append(newWebhooks, others...)
for i := range oldTargets {
currentTgt, ok := oldTargets[i].(*http.Target)
if !ok {
continue
}
var newTgt *http.Target
for ii := range tgts {
if currentTgt.Name() == tgts[ii].Name() {
newTgt = tgts[ii]
currentTgt.AssignMigrateTarget(newTgt)
http.CreateOrAdjustGlobalBuffer(currentTgt, newTgt)
break
}
}
}
for _, t := range tgts {
err := t.Init(ctx)
if err != nil {
errs = append(errs, err)
}
}
targetsList.set(newWebhooks)
cancelTargets(oldTargets)
return errs
}
// UpdateAuditKafkaTargets swaps audit kafka targets with newly loaded ones from the cfg
func UpdateAuditKafkaTargets(ctx context.Context, cfg Config) []error {
newKafkaTgts, errs := initKafkaTargets(ctx, cfg.AuditKafka)
// Retain webhook targets
oldKafkaTgts, otherTgts := splitTargets(auditTargets.get(), types.TargetKafka)
newKafkaTgts = append(newKafkaTgts, otherTgts...)
auditTargets.set(newKafkaTgts)
cancelTargets(oldKafkaTgts) // cancel running targets
return errs
}

69
internal/logger/utils.go Normal file
View File

@@ -0,0 +1,69 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package logger
import (
"context"
"errors"
"fmt"
"net/http"
"regexp"
"runtime"
"github.com/minio/minio/internal/color"
)
var ansiRE = regexp.MustCompile("(\x1b[^m]*m)")
// Print ANSI Control escape
func ansiEscape(format string, args ...any) {
Esc := "\x1b"
fmt.Printf("%s%s", Esc, fmt.Sprintf(format, args...))
}
func ansiMoveRight(n int) {
if runtime.GOOS == "windows" {
return
}
if color.IsTerminal() {
ansiEscape("[%dC", n)
}
}
func ansiSaveAttributes() {
if runtime.GOOS == "windows" {
return
}
if color.IsTerminal() {
ansiEscape("7")
}
}
func ansiRestoreAttributes() {
if runtime.GOOS == "windows" {
return
}
if color.IsTerminal() {
ansiEscape("8")
}
}
// logIgnoreError if true,the error will ignore.
func logIgnoreError(err error) bool {
return err == nil || errors.Is(err, context.Canceled) || errors.Is(err, http.ErrServerClosed) || err.Error() == "drive not found"
}