OP#792 initial commit after fork
This commit is contained in:
70
internal/hash/checker.go
Normal file
70
internal/hash/checker.go
Normal file
@@ -0,0 +1,70 @@
|
||||
// Copyright (c) 2015-2024 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package hash
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"hash"
|
||||
"io"
|
||||
|
||||
"github.com/minio/minio/internal/ioutil"
|
||||
)
|
||||
|
||||
// Checker allows to verify the checksum of a reader.
|
||||
type Checker struct {
|
||||
c io.Closer
|
||||
r io.Reader
|
||||
h hash.Hash
|
||||
|
||||
want []byte
|
||||
}
|
||||
|
||||
// NewChecker ensures that content with the specified length is read from rc.
|
||||
// Calling Close on this will close upstream.
|
||||
func NewChecker(rc io.ReadCloser, h hash.Hash, wantSum []byte, length int64) *Checker {
|
||||
return &Checker{c: rc, r: ioutil.HardLimitReader(rc, length), h: h, want: wantSum}
|
||||
}
|
||||
|
||||
// Read satisfies io.Reader
|
||||
func (c Checker) Read(p []byte) (n int, err error) {
|
||||
n, err = c.r.Read(p)
|
||||
if n > 0 {
|
||||
c.h.Write(p[:n])
|
||||
}
|
||||
if errors.Is(err, io.EOF) {
|
||||
got := c.h.Sum(nil)
|
||||
if !bytes.Equal(got, c.want) {
|
||||
return n, ErrInvalidChecksum
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
|
||||
// Close satisfies io.Closer
|
||||
func (c Checker) Close() error {
|
||||
err := c.c.Close()
|
||||
if err == nil {
|
||||
got := c.h.Sum(nil)
|
||||
if !bytes.Equal(got, c.want) {
|
||||
return ErrInvalidChecksum
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
752
internal/hash/checksum.go
Normal file
752
internal/hash/checksum.go
Normal file
@@ -0,0 +1,752 @@
|
||||
// Copyright (c) 2015-2022 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package hash
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha1"
|
||||
"encoding/base64"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"hash"
|
||||
"hash/crc32"
|
||||
"hash/crc64"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/minio/minio/internal/hash/sha256"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
)
|
||||
|
||||
func hashLogIf(ctx context.Context, err error) {
|
||||
logger.LogIf(ctx, "hash", err)
|
||||
}
|
||||
|
||||
// MinIOMultipartChecksum is as metadata on multipart uploads to indicate checksum type.
|
||||
const MinIOMultipartChecksum = "x-minio-multipart-checksum"
|
||||
|
||||
// MinIOMultipartChecksumType is as metadata on multipart uploads to indicate checksum type.
|
||||
const MinIOMultipartChecksumType = "x-minio-multipart-checksum-type"
|
||||
|
||||
// ChecksumType contains information about the checksum type.
|
||||
type ChecksumType uint32
|
||||
|
||||
const (
|
||||
|
||||
// ChecksumTrailing indicates the checksum will be sent in the trailing header.
|
||||
// Another checksum type will be set.
|
||||
ChecksumTrailing ChecksumType = 1 << iota
|
||||
|
||||
// ChecksumSHA256 indicates a SHA256 checksum.
|
||||
ChecksumSHA256
|
||||
// ChecksumSHA1 indicates a SHA-1 checksum.
|
||||
ChecksumSHA1
|
||||
// ChecksumCRC32 indicates a CRC32 checksum with IEEE table.
|
||||
ChecksumCRC32
|
||||
// ChecksumCRC32C indicates a CRC32 checksum with Castagnoli table.
|
||||
ChecksumCRC32C
|
||||
// ChecksumInvalid indicates an invalid checksum.
|
||||
ChecksumInvalid
|
||||
// ChecksumMultipart indicates the checksum is from a multipart upload.
|
||||
ChecksumMultipart
|
||||
// ChecksumIncludesMultipart indicates the checksum also contains part checksums.
|
||||
ChecksumIncludesMultipart
|
||||
// ChecksumCRC64NVME indicates CRC64 with 0xad93d23594c93659 polynomial.
|
||||
ChecksumCRC64NVME
|
||||
// ChecksumFullObject indicates the checksum is of the full object,
|
||||
// not checksum of checksums. Should only be set on ChecksumMultipart
|
||||
ChecksumFullObject
|
||||
|
||||
// ChecksumNone indicates no checksum.
|
||||
ChecksumNone ChecksumType = 0
|
||||
|
||||
baseTypeMask = ChecksumSHA256 | ChecksumSHA1 | ChecksumCRC32 | ChecksumCRC32C | ChecksumCRC64NVME
|
||||
)
|
||||
|
||||
// BaseChecksumTypes is a list of all the base checksum types.
|
||||
var BaseChecksumTypes = []ChecksumType{ChecksumSHA256, ChecksumSHA1, ChecksumCRC32, ChecksumCRC64NVME, ChecksumCRC32C}
|
||||
|
||||
// Checksum is a type and base 64 encoded value.
|
||||
type Checksum struct {
|
||||
Type ChecksumType
|
||||
Encoded string
|
||||
Raw []byte
|
||||
WantParts int
|
||||
}
|
||||
|
||||
// Is returns if c is all of t.
|
||||
func (c ChecksumType) Is(t ChecksumType) bool {
|
||||
if t == ChecksumNone {
|
||||
return c == ChecksumNone
|
||||
}
|
||||
return c&t == t
|
||||
}
|
||||
|
||||
// Base returns the base checksum (if any)
|
||||
func (c ChecksumType) Base() ChecksumType {
|
||||
return c & baseTypeMask
|
||||
}
|
||||
|
||||
// Key returns the header key.
|
||||
// returns empty string if invalid or none.
|
||||
func (c ChecksumType) Key() string {
|
||||
switch {
|
||||
case c.Is(ChecksumCRC32):
|
||||
return xhttp.AmzChecksumCRC32
|
||||
case c.Is(ChecksumCRC32C):
|
||||
return xhttp.AmzChecksumCRC32C
|
||||
case c.Is(ChecksumSHA1):
|
||||
return xhttp.AmzChecksumSHA1
|
||||
case c.Is(ChecksumSHA256):
|
||||
return xhttp.AmzChecksumSHA256
|
||||
case c.Is(ChecksumCRC64NVME):
|
||||
return xhttp.AmzChecksumCRC64NVME
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// RawByteLen returns the size of the un-encoded checksum.
|
||||
func (c ChecksumType) RawByteLen() int {
|
||||
switch {
|
||||
case c.Is(ChecksumCRC32):
|
||||
return 4
|
||||
case c.Is(ChecksumCRC32C):
|
||||
return 4
|
||||
case c.Is(ChecksumSHA1):
|
||||
return sha1.Size
|
||||
case c.Is(ChecksumSHA256):
|
||||
return sha256.Size
|
||||
case c.Is(ChecksumCRC64NVME):
|
||||
return crc64.Size
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// IsSet returns whether the type is valid and known.
|
||||
func (c ChecksumType) IsSet() bool {
|
||||
return !c.Is(ChecksumInvalid) && !c.Base().Is(ChecksumNone)
|
||||
}
|
||||
|
||||
// ChecksumStringToType is like NewChecksumType but without the `mode`
|
||||
func ChecksumStringToType(alg string) ChecksumType {
|
||||
switch strings.ToUpper(alg) {
|
||||
case "CRC32":
|
||||
return ChecksumCRC32
|
||||
case "CRC32C":
|
||||
return ChecksumCRC32C
|
||||
case "SHA1":
|
||||
return ChecksumSHA1
|
||||
case "SHA256":
|
||||
return ChecksumSHA256
|
||||
case "CRC64NVME":
|
||||
// AWS seems to ignore full value, and just assume it.
|
||||
return ChecksumCRC64NVME
|
||||
case "":
|
||||
return ChecksumNone
|
||||
}
|
||||
return ChecksumInvalid
|
||||
}
|
||||
|
||||
// NewChecksumType returns a checksum type based on the algorithm string and obj type.
|
||||
func NewChecksumType(alg, objType string) ChecksumType {
|
||||
full := ChecksumFullObject
|
||||
switch objType {
|
||||
case xhttp.AmzChecksumTypeFullObject:
|
||||
case xhttp.AmzChecksumTypeComposite, "":
|
||||
full = 0
|
||||
default:
|
||||
return ChecksumInvalid
|
||||
}
|
||||
|
||||
switch strings.ToUpper(alg) {
|
||||
case "CRC32":
|
||||
return ChecksumCRC32 | full
|
||||
case "CRC32C":
|
||||
return ChecksumCRC32C | full
|
||||
case "SHA1":
|
||||
if full != 0 {
|
||||
return ChecksumInvalid
|
||||
}
|
||||
return ChecksumSHA1
|
||||
case "SHA256":
|
||||
if full != 0 {
|
||||
return ChecksumInvalid
|
||||
}
|
||||
return ChecksumSHA256
|
||||
case "CRC64NVME":
|
||||
// AWS seems to ignore full value, and just assume it.
|
||||
return ChecksumCRC64NVME
|
||||
case "":
|
||||
if full != 0 {
|
||||
return ChecksumInvalid
|
||||
}
|
||||
return ChecksumNone
|
||||
}
|
||||
return ChecksumInvalid
|
||||
}
|
||||
|
||||
// NewChecksumHeader returns a checksum type based on the algorithm string.
|
||||
func NewChecksumHeader(h http.Header) ChecksumType {
|
||||
return NewChecksumType(h.Get(xhttp.AmzChecksumAlgo), h.Get(xhttp.AmzChecksumType))
|
||||
}
|
||||
|
||||
// String returns the type as a string.
|
||||
func (c ChecksumType) String() string {
|
||||
switch {
|
||||
case c.Is(ChecksumCRC32):
|
||||
return "CRC32"
|
||||
case c.Is(ChecksumCRC32C):
|
||||
return "CRC32C"
|
||||
case c.Is(ChecksumSHA1):
|
||||
return "SHA1"
|
||||
case c.Is(ChecksumSHA256):
|
||||
return "SHA256"
|
||||
case c.Is(ChecksumCRC64NVME):
|
||||
return "CRC64NVME"
|
||||
case c.Is(ChecksumNone):
|
||||
return ""
|
||||
}
|
||||
return "invalid"
|
||||
}
|
||||
|
||||
// StringFull returns the type and all flags as a string.
|
||||
func (c ChecksumType) StringFull() string {
|
||||
out := []string{c.String()}
|
||||
if c.Is(ChecksumMultipart) {
|
||||
out = append(out, "MULTIPART")
|
||||
}
|
||||
if c.Is(ChecksumIncludesMultipart) {
|
||||
out = append(out, "INCLUDESMP")
|
||||
}
|
||||
if c.Is(ChecksumTrailing) {
|
||||
out = append(out, "TRAILING")
|
||||
}
|
||||
if c.Is(ChecksumFullObject) {
|
||||
out = append(out, "FULLOBJ")
|
||||
}
|
||||
return strings.Join(out, "|")
|
||||
}
|
||||
|
||||
// FullObjectRequested will return if the checksum type indicates full object checksum was requested.
|
||||
func (c ChecksumType) FullObjectRequested() bool {
|
||||
return c&(ChecksumFullObject) == ChecksumFullObject || c.Is(ChecksumCRC64NVME)
|
||||
}
|
||||
|
||||
// IsMultipartComposite returns true if the checksum is multipart and full object was not requested.
|
||||
func (c ChecksumType) IsMultipartComposite() bool {
|
||||
return c.Is(ChecksumMultipart) && !c.FullObjectRequested()
|
||||
}
|
||||
|
||||
// ObjType returns a string to return as x-amz-checksum-type.
|
||||
func (c ChecksumType) ObjType() string {
|
||||
if c.FullObjectRequested() {
|
||||
return xhttp.AmzChecksumTypeFullObject
|
||||
}
|
||||
if c.IsMultipartComposite() {
|
||||
return xhttp.AmzChecksumTypeComposite
|
||||
}
|
||||
if !c.Is(ChecksumMultipart) {
|
||||
return xhttp.AmzChecksumTypeFullObject
|
||||
}
|
||||
if c.IsSet() {
|
||||
return xhttp.AmzChecksumTypeComposite
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// CanMerge will return if the checksum type indicates that checksums can be merged.
|
||||
func (c ChecksumType) CanMerge() bool {
|
||||
return c.Is(ChecksumCRC64NVME) || c.Is(ChecksumCRC32C) || c.Is(ChecksumCRC32)
|
||||
}
|
||||
|
||||
// Hasher returns a hasher corresponding to the checksum type.
|
||||
// Returns nil if no checksum.
|
||||
func (c ChecksumType) Hasher() hash.Hash {
|
||||
switch {
|
||||
case c.Is(ChecksumCRC32):
|
||||
return crc32.NewIEEE()
|
||||
case c.Is(ChecksumCRC32C):
|
||||
return crc32.New(crc32.MakeTable(crc32.Castagnoli))
|
||||
case c.Is(ChecksumSHA1):
|
||||
return sha1.New()
|
||||
case c.Is(ChecksumSHA256):
|
||||
return sha256.New()
|
||||
case c.Is(ChecksumCRC64NVME):
|
||||
return crc64.New(crc64Table)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Trailing return whether the checksum is trailing.
|
||||
func (c ChecksumType) Trailing() bool {
|
||||
return c.Is(ChecksumTrailing)
|
||||
}
|
||||
|
||||
// NewChecksumFromData returns a new Checksum, using specified algorithm type on data.
|
||||
func NewChecksumFromData(t ChecksumType, data []byte) *Checksum {
|
||||
if !t.IsSet() {
|
||||
return nil
|
||||
}
|
||||
h := t.Hasher()
|
||||
h.Write(data)
|
||||
raw := h.Sum(nil)
|
||||
c := Checksum{Type: t, Encoded: base64.StdEncoding.EncodeToString(raw), Raw: raw}
|
||||
if !c.Valid() {
|
||||
return nil
|
||||
}
|
||||
return &c
|
||||
}
|
||||
|
||||
// ReadCheckSums will read checksums from b and return them.
|
||||
// Returns whether this is (part of) a multipart checksum.
|
||||
func ReadCheckSums(b []byte, part int) (cs map[string]string, isMP bool) {
|
||||
res := make(map[string]string, 1)
|
||||
for len(b) > 0 {
|
||||
t, n := binary.Uvarint(b)
|
||||
if n < 0 {
|
||||
break
|
||||
}
|
||||
b = b[n:]
|
||||
|
||||
typ := ChecksumType(t)
|
||||
length := typ.RawByteLen()
|
||||
if length == 0 || len(b) < length {
|
||||
break
|
||||
}
|
||||
|
||||
cs := base64.StdEncoding.EncodeToString(b[:length])
|
||||
b = b[length:]
|
||||
if typ.Is(ChecksumMultipart) {
|
||||
isMP = true
|
||||
t, n := binary.Uvarint(b)
|
||||
if n < 0 {
|
||||
break
|
||||
}
|
||||
if !typ.FullObjectRequested() {
|
||||
cs = fmt.Sprintf("%s-%d", cs, t)
|
||||
}
|
||||
b = b[n:]
|
||||
if part > 0 {
|
||||
cs = ""
|
||||
}
|
||||
if typ.Is(ChecksumIncludesMultipart) {
|
||||
wantLen := int(t) * length
|
||||
if len(b) < wantLen {
|
||||
break
|
||||
}
|
||||
// Read part checksum
|
||||
if part > 0 && uint64(part) <= t {
|
||||
offset := (part - 1) * length
|
||||
partCs := b[offset:]
|
||||
cs = base64.StdEncoding.EncodeToString(partCs[:length])
|
||||
}
|
||||
b = b[wantLen:]
|
||||
}
|
||||
} else if part > 1 {
|
||||
// For non-multipart, checksum is part 1.
|
||||
cs = ""
|
||||
}
|
||||
if cs != "" {
|
||||
res[typ.String()] = cs
|
||||
if ckType := typ.ObjType(); ckType != "" {
|
||||
res[xhttp.AmzChecksumType] = ckType
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(res) == 0 {
|
||||
res = nil
|
||||
}
|
||||
return res, isMP
|
||||
}
|
||||
|
||||
// ReadPartCheckSums will read all part checksums from b and return them.
|
||||
func ReadPartCheckSums(b []byte) (res []map[string]string) {
|
||||
for len(b) > 0 {
|
||||
t, n := binary.Uvarint(b)
|
||||
if n <= 0 {
|
||||
break
|
||||
}
|
||||
b = b[n:]
|
||||
|
||||
typ := ChecksumType(t)
|
||||
length := typ.RawByteLen()
|
||||
if length == 0 || len(b) < length {
|
||||
break
|
||||
}
|
||||
// Skip main checksum
|
||||
b = b[length:]
|
||||
parts, n := binary.Uvarint(b)
|
||||
if n <= 0 {
|
||||
break
|
||||
}
|
||||
if !typ.Is(ChecksumIncludesMultipart) {
|
||||
continue
|
||||
}
|
||||
|
||||
if len(res) == 0 {
|
||||
res = make([]map[string]string, parts)
|
||||
}
|
||||
b = b[n:]
|
||||
for part := 0; part < int(parts); part++ {
|
||||
if len(b) < length {
|
||||
break
|
||||
}
|
||||
// Read part checksum
|
||||
cs := base64.StdEncoding.EncodeToString(b[:length])
|
||||
b = b[length:]
|
||||
if res[part] == nil {
|
||||
res[part] = make(map[string]string, 1)
|
||||
}
|
||||
res[part][typ.String()] = cs
|
||||
}
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
// NewChecksumWithType is similar to NewChecksumString but expects input algo of ChecksumType.
|
||||
func NewChecksumWithType(alg ChecksumType, value string) *Checksum {
|
||||
if !alg.IsSet() {
|
||||
return nil
|
||||
}
|
||||
wantParts := 0
|
||||
if strings.ContainsRune(value, '-') {
|
||||
valSplit := strings.Split(value, "-")
|
||||
if len(valSplit) != 2 {
|
||||
return nil
|
||||
}
|
||||
value = valSplit[0]
|
||||
nParts, err := strconv.Atoi(valSplit[1])
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
alg |= ChecksumMultipart
|
||||
wantParts = nParts
|
||||
}
|
||||
bvalue, err := base64.StdEncoding.DecodeString(value)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
c := Checksum{Type: alg, Encoded: value, Raw: bvalue, WantParts: wantParts}
|
||||
if !c.Valid() {
|
||||
return nil
|
||||
}
|
||||
return &c
|
||||
}
|
||||
|
||||
// NewChecksumString returns a new checksum from specified algorithm and base64 encoded value.
|
||||
func NewChecksumString(alg, value string) *Checksum {
|
||||
return NewChecksumWithType(NewChecksumType(alg, ""), value)
|
||||
}
|
||||
|
||||
// AppendTo will append the checksum to b.
|
||||
// 'parts' is used when checksum has ChecksumMultipart set.
|
||||
// ReadCheckSums reads the values back.
|
||||
func (c *Checksum) AppendTo(b []byte, parts []byte) []byte {
|
||||
if c == nil {
|
||||
return nil
|
||||
}
|
||||
var tmp [binary.MaxVarintLen32]byte
|
||||
n := binary.PutUvarint(tmp[:], uint64(c.Type))
|
||||
crc := c.Raw
|
||||
if c.Type.Trailing() {
|
||||
// When we serialize we don't care if it was trailing.
|
||||
c.Type ^= ChecksumTrailing
|
||||
}
|
||||
if len(crc) != c.Type.RawByteLen() {
|
||||
return b
|
||||
}
|
||||
b = append(b, tmp[:n]...)
|
||||
b = append(b, crc...)
|
||||
if c.Type.Is(ChecksumMultipart) {
|
||||
var checksums int
|
||||
if c.WantParts > 0 && !c.Type.Is(ChecksumIncludesMultipart) {
|
||||
checksums = c.WantParts
|
||||
}
|
||||
// Ensure we don't divide by 0:
|
||||
if c.Type.RawByteLen() == 0 || len(parts)%c.Type.RawByteLen() != 0 {
|
||||
hashLogIf(context.Background(), fmt.Errorf("internal error: Unexpected checksum length: %d, each checksum %d", len(parts), c.Type.RawByteLen()))
|
||||
checksums = 0
|
||||
parts = nil
|
||||
} else if len(parts) > 0 {
|
||||
checksums = len(parts) / c.Type.RawByteLen()
|
||||
}
|
||||
if !c.Type.Is(ChecksumIncludesMultipart) {
|
||||
parts = nil
|
||||
}
|
||||
n := binary.PutUvarint(tmp[:], uint64(checksums))
|
||||
b = append(b, tmp[:n]...)
|
||||
if len(parts) > 0 {
|
||||
b = append(b, parts...)
|
||||
}
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// ChecksumFromBytes reconstructs a Checksum struct from the serialized bytes created in AppendTo()
|
||||
// Returns nil if the bytes are invalid or empty.
|
||||
// AppendTo() can append a serialized Checksum to another already-serialized Checksum,
|
||||
// however, in practice, we only use one at a time.
|
||||
// ChecksumFromBytes only returns the first one and no part checksums.
|
||||
func ChecksumFromBytes(b []byte) *Checksum {
|
||||
if len(b) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Read checksum type
|
||||
t, n := binary.Uvarint(b)
|
||||
if n <= 0 {
|
||||
return nil
|
||||
}
|
||||
b = b[n:]
|
||||
|
||||
typ := ChecksumType(t)
|
||||
length := typ.RawByteLen()
|
||||
if length == 0 || len(b) < length {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Read raw checksum bytes
|
||||
raw := make([]byte, length)
|
||||
copy(raw, b[:length])
|
||||
b = b[length:]
|
||||
|
||||
c := &Checksum{
|
||||
Type: typ,
|
||||
Raw: raw,
|
||||
Encoded: base64.StdEncoding.EncodeToString(raw),
|
||||
}
|
||||
|
||||
// Handle multipart checksums
|
||||
if typ.Is(ChecksumMultipart) {
|
||||
parts, n := binary.Uvarint(b)
|
||||
if n <= 0 {
|
||||
return nil
|
||||
}
|
||||
b = b[n:]
|
||||
|
||||
c.WantParts = int(parts)
|
||||
|
||||
if typ.Is(ChecksumIncludesMultipart) {
|
||||
wantLen := int(parts) * length
|
||||
if len(b) < wantLen {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !c.Valid() {
|
||||
return nil
|
||||
}
|
||||
|
||||
return c
|
||||
}
|
||||
|
||||
// Valid returns whether checksum is valid.
|
||||
func (c Checksum) Valid() bool {
|
||||
if c.Type == ChecksumInvalid {
|
||||
return false
|
||||
}
|
||||
if len(c.Encoded) == 0 || c.Type.Trailing() {
|
||||
return c.Type.Is(ChecksumNone) || c.Type.Trailing()
|
||||
}
|
||||
return c.Type.RawByteLen() == len(c.Raw)
|
||||
}
|
||||
|
||||
// Matches returns whether given content matches c.
|
||||
func (c Checksum) Matches(content []byte, parts int) error {
|
||||
if len(c.Encoded) == 0 {
|
||||
return nil
|
||||
}
|
||||
hasher := c.Type.Hasher()
|
||||
_, err := hasher.Write(content)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sum := hasher.Sum(nil)
|
||||
if c.WantParts > 0 && c.WantParts != parts {
|
||||
return ChecksumMismatch{
|
||||
Want: fmt.Sprintf("%s-%d", c.Encoded, c.WantParts),
|
||||
Got: fmt.Sprintf("%s-%d", base64.StdEncoding.EncodeToString(sum), parts),
|
||||
}
|
||||
}
|
||||
|
||||
if !bytes.Equal(sum, c.Raw) {
|
||||
return ChecksumMismatch{
|
||||
Want: c.Encoded,
|
||||
Got: base64.StdEncoding.EncodeToString(sum),
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// AsMap returns the checksum as a map[string]string.
|
||||
func (c *Checksum) AsMap() map[string]string {
|
||||
if c == nil || !c.Valid() {
|
||||
return nil
|
||||
}
|
||||
return map[string]string{
|
||||
c.Type.String(): c.Encoded,
|
||||
xhttp.AmzChecksumType: c.Type.ObjType(),
|
||||
}
|
||||
}
|
||||
|
||||
// Equal returns whether two checksum structs are equal in all their fields.
|
||||
func (c *Checksum) Equal(s *Checksum) bool {
|
||||
if c == nil || s == nil {
|
||||
return c == s
|
||||
}
|
||||
return c.Type == s.Type &&
|
||||
c.Encoded == s.Encoded &&
|
||||
bytes.Equal(c.Raw, s.Raw) &&
|
||||
c.WantParts == s.WantParts
|
||||
}
|
||||
|
||||
// TransferChecksumHeader will transfer any checksum value that has been checked.
|
||||
// If checksum was trailing, they must have been added to r.Trailer.
|
||||
func TransferChecksumHeader(w http.ResponseWriter, r *http.Request) {
|
||||
c, err := GetContentChecksum(r.Header)
|
||||
if err != nil || c == nil {
|
||||
return
|
||||
}
|
||||
t, s := c.Type, c.Encoded
|
||||
if !c.Type.IsSet() {
|
||||
return
|
||||
}
|
||||
if c.Type.Is(ChecksumTrailing) {
|
||||
val := r.Trailer.Get(t.Key())
|
||||
if val != "" {
|
||||
w.Header().Set(t.Key(), val)
|
||||
}
|
||||
return
|
||||
}
|
||||
w.Header().Set(t.Key(), s)
|
||||
}
|
||||
|
||||
// AddChecksumHeader will transfer any checksum value that has been checked.
|
||||
func AddChecksumHeader(w http.ResponseWriter, c map[string]string) {
|
||||
for k, v := range c {
|
||||
if k == xhttp.AmzChecksumType {
|
||||
w.Header().Set(xhttp.AmzChecksumType, v)
|
||||
continue
|
||||
}
|
||||
cksum := NewChecksumString(k, v)
|
||||
if cksum == nil {
|
||||
continue
|
||||
}
|
||||
if cksum.Valid() {
|
||||
w.Header().Set(cksum.Type.Key(), v)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// GetContentChecksum returns content checksum.
|
||||
// Returns ErrInvalidChecksum if so.
|
||||
// Returns nil, nil if no checksum.
|
||||
func GetContentChecksum(h http.Header) (*Checksum, error) {
|
||||
if trailing := h.Values(xhttp.AmzTrailer); len(trailing) > 0 {
|
||||
var res *Checksum
|
||||
for _, header := range trailing {
|
||||
var duplicates bool
|
||||
for _, t := range BaseChecksumTypes {
|
||||
if strings.EqualFold(t.Key(), header) {
|
||||
duplicates = res != nil
|
||||
res = NewChecksumWithType(t|ChecksumTrailing, "")
|
||||
}
|
||||
}
|
||||
if duplicates {
|
||||
return nil, ErrInvalidChecksum
|
||||
}
|
||||
}
|
||||
if res != nil {
|
||||
switch h.Get(xhttp.AmzChecksumType) {
|
||||
case xhttp.AmzChecksumTypeFullObject:
|
||||
if !res.Type.CanMerge() {
|
||||
return nil, ErrInvalidChecksum
|
||||
}
|
||||
res.Type |= ChecksumFullObject
|
||||
case xhttp.AmzChecksumTypeComposite, "":
|
||||
default:
|
||||
return nil, ErrInvalidChecksum
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
}
|
||||
t, s := getContentChecksum(h)
|
||||
if t == ChecksumNone {
|
||||
if s == "" {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, ErrInvalidChecksum
|
||||
}
|
||||
cksum := NewChecksumWithType(t, s)
|
||||
if cksum == nil {
|
||||
return nil, ErrInvalidChecksum
|
||||
}
|
||||
return cksum, nil
|
||||
}
|
||||
|
||||
// getContentChecksum returns content checksum type and value.
|
||||
// Returns ChecksumInvalid if so.
|
||||
func getContentChecksum(h http.Header) (t ChecksumType, s string) {
|
||||
t = ChecksumNone
|
||||
alg := h.Get(xhttp.AmzChecksumAlgo)
|
||||
if alg != "" {
|
||||
t |= NewChecksumHeader(h)
|
||||
if h.Get(xhttp.AmzChecksumType) == xhttp.AmzChecksumTypeFullObject {
|
||||
if !t.CanMerge() {
|
||||
return ChecksumInvalid, ""
|
||||
}
|
||||
t |= ChecksumFullObject
|
||||
}
|
||||
if t.IsSet() {
|
||||
hdr := t.Key()
|
||||
if s = h.Get(hdr); s == "" {
|
||||
return ChecksumNone, ""
|
||||
}
|
||||
}
|
||||
return t, s
|
||||
}
|
||||
checkType := func(c ChecksumType) {
|
||||
if got := h.Get(c.Key()); got != "" {
|
||||
// If already set, invalid
|
||||
if t != ChecksumNone {
|
||||
t = ChecksumInvalid
|
||||
s = ""
|
||||
} else {
|
||||
t = c
|
||||
s = got
|
||||
}
|
||||
if h.Get(xhttp.AmzChecksumType) == xhttp.AmzChecksumTypeFullObject {
|
||||
if !t.CanMerge() {
|
||||
t = ChecksumInvalid
|
||||
s = ""
|
||||
return
|
||||
}
|
||||
t |= ChecksumFullObject
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
for _, t := range BaseChecksumTypes {
|
||||
checkType(t)
|
||||
}
|
||||
return t, s
|
||||
}
|
||||
205
internal/hash/checksum_test.go
Normal file
205
internal/hash/checksum_test.go
Normal file
@@ -0,0 +1,205 @@
|
||||
// Copyright (c) 2015-2025 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package hash
|
||||
|
||||
import (
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
)
|
||||
|
||||
// TestChecksumAddToHeader tests that adding and retrieving a checksum on a header works
|
||||
func TestChecksumAddToHeader(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
checksum ChecksumType
|
||||
fullobj bool
|
||||
wantErr bool
|
||||
}{
|
||||
{"CRC32-composite", ChecksumCRC32, false, false},
|
||||
{"CRC32-full-object", ChecksumCRC32, true, false},
|
||||
{"CRC32C-composite", ChecksumCRC32C, false, false},
|
||||
{"CRC32C-full-object", ChecksumCRC32C, true, false},
|
||||
{"CRC64NVME-full-object", ChecksumCRC64NVME, false, false}, // CRC64NVME is always full object
|
||||
{"ChecksumSHA1-composite", ChecksumSHA1, false, false},
|
||||
{"ChecksumSHA256-composite", ChecksumSHA256, false, false},
|
||||
{"ChecksumSHA1-full-object", ChecksumSHA1, true, true}, // SHA1 does not support full object
|
||||
{"ChecksumSHA256-full-object", ChecksumSHA256, true, true}, // SHA256 does not support full object
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
// Skip invalid cases where SHA1 or SHA256 is used with full object
|
||||
if (tt.checksum.Is(ChecksumSHA1) || tt.checksum.Is(ChecksumSHA256)) && tt.fullobj {
|
||||
// Validate that NewChecksumType correctly marks these as invalid
|
||||
alg := tt.checksum.String()
|
||||
typ := NewChecksumType(alg, xhttp.AmzChecksumTypeFullObject)
|
||||
if !typ.Is(ChecksumInvalid) {
|
||||
t.Fatalf("Expected ChecksumInvalid for %s with full object, got %s", tt.name, typ.StringFull())
|
||||
}
|
||||
return
|
||||
}
|
||||
myData := []byte("this-is-a-checksum-data-test")
|
||||
chksm := NewChecksumFromData(tt.checksum, myData)
|
||||
if chksm == nil {
|
||||
t.Fatalf("NewChecksumFromData failed for %s", tt.name)
|
||||
}
|
||||
if tt.fullobj {
|
||||
chksm.Type |= ChecksumFullObject
|
||||
}
|
||||
|
||||
// CRC64NVME is always full object
|
||||
if chksm.Type.Base().Is(ChecksumCRC64NVME) {
|
||||
chksm.Type |= ChecksumFullObject
|
||||
}
|
||||
|
||||
// Prepare the checksum map with appropriate headers
|
||||
m := chksm.AsMap()
|
||||
m[xhttp.AmzChecksumAlgo] = chksm.Type.String() // Set the algorithm explicitly
|
||||
if chksm.Type.FullObjectRequested() {
|
||||
m[xhttp.AmzChecksumType] = xhttp.AmzChecksumTypeFullObject
|
||||
} else {
|
||||
m[xhttp.AmzChecksumType] = xhttp.AmzChecksumTypeComposite
|
||||
}
|
||||
|
||||
w := httptest.NewRecorder()
|
||||
AddChecksumHeader(w, m)
|
||||
gotChksm, err := GetContentChecksum(w.Result().Header)
|
||||
if tt.wantErr {
|
||||
if err == nil {
|
||||
t.Fatalf("Expected error for %s, got none", tt.name)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("GetContentChecksum failed for %s: %v", tt.name, err)
|
||||
}
|
||||
|
||||
if gotChksm == nil {
|
||||
t.Fatalf("Got nil checksum for %s", tt.name)
|
||||
}
|
||||
// Compare the full checksum structs
|
||||
if !chksm.Equal(gotChksm) {
|
||||
t.Errorf("Checksum mismatch for %s: expected %+v, got %+v", tt.name, chksm, gotChksm)
|
||||
}
|
||||
// Verify the checksum type
|
||||
expectedType := chksm.Type
|
||||
if gotChksm.Type != expectedType {
|
||||
t.Errorf("Type mismatch for %s: expected %s, got %s", tt.name, expectedType.StringFull(), gotChksm.Type.StringFull())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestChecksumSerializeDeserialize checks AppendTo can be reversed by ChecksumFromBytes
|
||||
func TestChecksumSerializeDeserialize(t *testing.T) {
|
||||
myData := []byte("this-is-a-checksum-data-test")
|
||||
chksm := NewChecksumFromData(ChecksumCRC32, myData)
|
||||
if chksm == nil {
|
||||
t.Fatal("NewChecksumFromData returned nil")
|
||||
}
|
||||
// Serialize the checksum to bytes
|
||||
b := chksm.AppendTo(nil, nil)
|
||||
if b == nil {
|
||||
t.Fatal("AppendTo returned nil")
|
||||
}
|
||||
|
||||
// Deserialize the checksum from bytes
|
||||
chksmOut := ChecksumFromBytes(b)
|
||||
if chksmOut == nil {
|
||||
t.Fatal("ChecksumFromBytes returned nil")
|
||||
}
|
||||
|
||||
// Assert new checksum matches the content
|
||||
matchError := chksmOut.Matches(myData, 0)
|
||||
if matchError != nil {
|
||||
t.Fatalf("Checksum mismatch on chksmOut: %v", matchError)
|
||||
}
|
||||
|
||||
// Assert they are exactly equal
|
||||
if !chksmOut.Equal(chksm) {
|
||||
t.Fatalf("Checksum mismatch: expected %+v, got %+v", chksm, chksmOut)
|
||||
}
|
||||
}
|
||||
|
||||
// TestChecksumSerializeDeserializeMultiPart checks AppendTo can be reversed by ChecksumFromBytes
|
||||
// for multipart checksum
|
||||
func TestChecksumSerializeDeserializeMultiPart(t *testing.T) {
|
||||
// Create dummy data that we'll split into 3 parts
|
||||
dummyData := []byte("The quick brown fox jumps over the lazy dog. " +
|
||||
"Pack my box with five dozen brown eggs. " +
|
||||
"Have another go it will all make sense in the end!")
|
||||
|
||||
// Split data into 3 parts
|
||||
partSize := len(dummyData) / 3
|
||||
part1Data := dummyData[0:partSize]
|
||||
part2Data := dummyData[partSize : 2*partSize]
|
||||
part3Data := dummyData[2*partSize:]
|
||||
|
||||
// Calculate CRC32C checksum for each part using NewChecksumFromData
|
||||
checksumType := ChecksumCRC32C
|
||||
|
||||
part1Checksum := NewChecksumFromData(checksumType, part1Data)
|
||||
part2Checksum := NewChecksumFromData(checksumType, part2Data)
|
||||
part3Checksum := NewChecksumFromData(checksumType, part3Data)
|
||||
|
||||
// Combine the raw checksums (this is what happens in CompleteMultipartUpload)
|
||||
var checksumCombined []byte
|
||||
checksumCombined = append(checksumCombined, part1Checksum.Raw...)
|
||||
checksumCombined = append(checksumCombined, part2Checksum.Raw...)
|
||||
checksumCombined = append(checksumCombined, part3Checksum.Raw...)
|
||||
|
||||
// Create the final checksum (checksum of the combined checksums)
|
||||
// Add BOTH the multipart flag AND the includes-multipart flag
|
||||
finalChecksumType := checksumType | ChecksumMultipart | ChecksumIncludesMultipart
|
||||
finalChecksum := NewChecksumFromData(finalChecksumType, checksumCombined)
|
||||
|
||||
// Set WantParts to indicate 3 parts
|
||||
finalChecksum.WantParts = 3
|
||||
|
||||
// Test AppendTo serialization
|
||||
var serialized []byte
|
||||
serialized = finalChecksum.AppendTo(serialized, checksumCombined)
|
||||
|
||||
// Use ChecksumFromBytes to deserialize the final checksum
|
||||
chksmOut := ChecksumFromBytes(serialized)
|
||||
if chksmOut == nil {
|
||||
t.Fatal("ChecksumFromBytes returned nil")
|
||||
}
|
||||
|
||||
// Assert they are exactly equal
|
||||
if !chksmOut.Equal(finalChecksum) {
|
||||
t.Fatalf("Checksum mismatch: expected %+v, got %+v", finalChecksum, chksmOut)
|
||||
}
|
||||
|
||||
// Serialize what we got from ChecksumFromBytes
|
||||
serializedOut := chksmOut.AppendTo(nil, checksumCombined)
|
||||
|
||||
// Read part checksums from serializedOut
|
||||
readParts := ReadPartCheckSums(serializedOut)
|
||||
expectedChecksums := []string{
|
||||
part1Checksum.Encoded,
|
||||
part2Checksum.Encoded,
|
||||
part3Checksum.Encoded,
|
||||
}
|
||||
for i, expected := range expectedChecksums {
|
||||
if got := readParts[i][ChecksumCRC32C.String()]; got != expected {
|
||||
t.Fatalf("want part%dChecksum.Encoded %s, got %s", i+1, expected, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
219
internal/hash/crc.go
Normal file
219
internal/hash/crc.go
Normal file
@@ -0,0 +1,219 @@
|
||||
// Copyright (c) 2015-2024 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package hash
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"hash/crc32"
|
||||
"hash/crc64"
|
||||
"math/bits"
|
||||
)
|
||||
|
||||
// AddPart will merge a part checksum into the current,
|
||||
// as if the content of each was appended.
|
||||
// The size of the content that produced the second checksum must be provided.
|
||||
// Not all checksum types can be merged, use the CanMerge method to check.
|
||||
// Checksum types must match.
|
||||
func (c *Checksum) AddPart(other Checksum, size int64) error {
|
||||
if !other.Type.CanMerge() {
|
||||
return fmt.Errorf("checksum type cannot be merged")
|
||||
}
|
||||
if size == 0 {
|
||||
return nil
|
||||
}
|
||||
if !c.Type.Is(other.Type.Base()) {
|
||||
return fmt.Errorf("checksum type does not match got %s and %s", c.Type.String(), other.Type.String())
|
||||
}
|
||||
// If never set, just add first checksum.
|
||||
if len(c.Raw) == 0 {
|
||||
c.Raw = other.Raw
|
||||
c.Encoded = other.Encoded
|
||||
return nil
|
||||
}
|
||||
if !c.Valid() {
|
||||
return fmt.Errorf("invalid base checksum")
|
||||
}
|
||||
if !other.Valid() {
|
||||
return fmt.Errorf("invalid part checksum")
|
||||
}
|
||||
|
||||
switch c.Type.Base() {
|
||||
case ChecksumCRC32:
|
||||
v := crc32Combine(crc32.IEEE, binary.BigEndian.Uint32(c.Raw), binary.BigEndian.Uint32(other.Raw), size)
|
||||
binary.BigEndian.PutUint32(c.Raw, v)
|
||||
case ChecksumCRC32C:
|
||||
v := crc32Combine(crc32.Castagnoli, binary.BigEndian.Uint32(c.Raw), binary.BigEndian.Uint32(other.Raw), size)
|
||||
binary.BigEndian.PutUint32(c.Raw, v)
|
||||
case ChecksumCRC64NVME:
|
||||
v := crc64Combine(bits.Reverse64(crc64NVMEPolynomial), binary.BigEndian.Uint64(c.Raw), binary.BigEndian.Uint64(other.Raw), size)
|
||||
binary.BigEndian.PutUint64(c.Raw, v)
|
||||
default:
|
||||
return fmt.Errorf("unknown checksum type: %s", c.Type.String())
|
||||
}
|
||||
c.Encoded = base64.StdEncoding.EncodeToString(c.Raw)
|
||||
return nil
|
||||
}
|
||||
|
||||
const crc64NVMEPolynomial = 0xad93d23594c93659
|
||||
|
||||
var crc64Table = crc64.MakeTable(bits.Reverse64(crc64NVMEPolynomial))
|
||||
|
||||
// Following is ported from C to Go in 2016 by Justin Ruggles, with minimal alteration.
|
||||
// Used uint for unsigned long. Used uint32 for input arguments in order to match
|
||||
// the Go hash/crc32 package. zlib CRC32 combine (https://github.com/madler/zlib)
|
||||
// Modified for hash/crc64 by Klaus Post, 2024.
|
||||
func gf2MatrixTimes(mat []uint64, vec uint64) uint64 {
|
||||
var sum uint64
|
||||
|
||||
for vec != 0 {
|
||||
if vec&1 != 0 {
|
||||
sum ^= mat[0]
|
||||
}
|
||||
vec >>= 1
|
||||
mat = mat[1:]
|
||||
}
|
||||
return sum
|
||||
}
|
||||
|
||||
func gf2MatrixSquare(square, mat []uint64) {
|
||||
if len(square) != len(mat) {
|
||||
panic("square matrix size mismatch")
|
||||
}
|
||||
for n := range mat {
|
||||
square[n] = gf2MatrixTimes(mat, mat[n])
|
||||
}
|
||||
}
|
||||
|
||||
// crc32Combine returns the combined CRC-32 hash value of the two passed CRC-32
|
||||
// hash values crc1 and crc2. poly represents the generator polynomial
|
||||
// and len2 specifies the byte length that the crc2 hash covers.
|
||||
func crc32Combine(poly uint32, crc1, crc2 uint32, len2 int64) uint32 {
|
||||
// degenerate case (also disallow negative lengths)
|
||||
if len2 <= 0 {
|
||||
return crc1
|
||||
}
|
||||
|
||||
even := make([]uint64, 32) // even-power-of-two zeros operator
|
||||
odd := make([]uint64, 32) // odd-power-of-two zeros operator
|
||||
|
||||
// put operator for one zero bit in odd
|
||||
odd[0] = uint64(poly) // CRC-32 polynomial
|
||||
row := uint64(1)
|
||||
for n := 1; n < 32; n++ {
|
||||
odd[n] = row
|
||||
row <<= 1
|
||||
}
|
||||
|
||||
// put operator for two zero bits in even
|
||||
gf2MatrixSquare(even, odd)
|
||||
|
||||
// put operator for four zero bits in odd
|
||||
gf2MatrixSquare(odd, even)
|
||||
|
||||
// apply len2 zeros to crc1 (first square will put the operator for one
|
||||
// zero byte, eight zero bits, in even)
|
||||
crc1n := uint64(crc1)
|
||||
for {
|
||||
// apply zeros operator for this bit of len2
|
||||
gf2MatrixSquare(even, odd)
|
||||
if len2&1 != 0 {
|
||||
crc1n = gf2MatrixTimes(even, crc1n)
|
||||
}
|
||||
len2 >>= 1
|
||||
|
||||
// if no more bits set, then done
|
||||
if len2 == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
// another iteration of the loop with odd and even swapped
|
||||
gf2MatrixSquare(odd, even)
|
||||
if len2&1 != 0 {
|
||||
crc1n = gf2MatrixTimes(odd, crc1n)
|
||||
}
|
||||
len2 >>= 1
|
||||
|
||||
// if no more bits set, then done
|
||||
if len2 == 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// return combined crc
|
||||
crc1n ^= uint64(crc2)
|
||||
return uint32(crc1n)
|
||||
}
|
||||
|
||||
func crc64Combine(poly uint64, crc1, crc2 uint64, len2 int64) uint64 {
|
||||
// degenerate case (also disallow negative lengths)
|
||||
if len2 <= 0 {
|
||||
return crc1
|
||||
}
|
||||
|
||||
even := make([]uint64, 64) // even-power-of-two zeros operator
|
||||
odd := make([]uint64, 64) // odd-power-of-two zeros operator
|
||||
|
||||
// put operator for one zero bit in odd
|
||||
odd[0] = poly // CRC-64 polynomial
|
||||
row := uint64(1)
|
||||
for n := 1; n < 64; n++ {
|
||||
odd[n] = row
|
||||
row <<= 1
|
||||
}
|
||||
|
||||
// put operator for two zero bits in even
|
||||
gf2MatrixSquare(even, odd)
|
||||
|
||||
// put operator for four zero bits in odd
|
||||
gf2MatrixSquare(odd, even)
|
||||
|
||||
// apply len2 zeros to crc1 (first square will put the operator for one
|
||||
// zero byte, eight zero bits, in even)
|
||||
crc1n := crc1
|
||||
for {
|
||||
// apply zeros operator for this bit of len2
|
||||
gf2MatrixSquare(even, odd)
|
||||
if len2&1 != 0 {
|
||||
crc1n = gf2MatrixTimes(even, crc1n)
|
||||
}
|
||||
len2 >>= 1
|
||||
|
||||
// if no more bits set, then done
|
||||
if len2 == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
// another iteration of the loop with odd and even swapped
|
||||
gf2MatrixSquare(odd, even)
|
||||
if len2&1 != 0 {
|
||||
crc1n = gf2MatrixTimes(odd, crc1n)
|
||||
}
|
||||
len2 >>= 1
|
||||
|
||||
// if no more bits set, then done
|
||||
if len2 == 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// return combined crc
|
||||
crc1n ^= crc2
|
||||
return crc1n
|
||||
}
|
||||
89
internal/hash/errors.go
Normal file
89
internal/hash/errors.go
Normal file
@@ -0,0 +1,89 @@
|
||||
// Copyright (c) 2015-2021 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package hash
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// SHA256Mismatch - when content sha256 does not match with what was sent from client.
|
||||
type SHA256Mismatch struct {
|
||||
ExpectedSHA256 string
|
||||
CalculatedSHA256 string
|
||||
}
|
||||
|
||||
func (e SHA256Mismatch) Error() string {
|
||||
return "Bad sha256: Expected " + e.ExpectedSHA256 + " does not match calculated " + e.CalculatedSHA256
|
||||
}
|
||||
|
||||
// BadDigest - Content-MD5 you specified did not match what we received.
|
||||
type BadDigest struct {
|
||||
ExpectedMD5 string
|
||||
CalculatedMD5 string
|
||||
}
|
||||
|
||||
func (e BadDigest) Error() string {
|
||||
return "Bad digest: Expected " + e.ExpectedMD5 + " does not match calculated " + e.CalculatedMD5
|
||||
}
|
||||
|
||||
// SizeTooSmall reader size too small
|
||||
type SizeTooSmall struct {
|
||||
Want int64
|
||||
Got int64
|
||||
}
|
||||
|
||||
func (e SizeTooSmall) Error() string {
|
||||
return fmt.Sprintf("Size small: got %d, want %d", e.Got, e.Want)
|
||||
}
|
||||
|
||||
// SizeTooLarge reader size too large
|
||||
type SizeTooLarge struct {
|
||||
Want int64
|
||||
Got int64
|
||||
}
|
||||
|
||||
func (e SizeTooLarge) Error() string {
|
||||
return fmt.Sprintf("Size large: got %d, want %d", e.Got, e.Want)
|
||||
}
|
||||
|
||||
// SizeMismatch error size mismatch
|
||||
type SizeMismatch struct {
|
||||
Want int64
|
||||
Got int64
|
||||
}
|
||||
|
||||
func (e SizeMismatch) Error() string {
|
||||
return fmt.Sprintf("Size mismatch: got %d, want %d", e.Got, e.Want)
|
||||
}
|
||||
|
||||
// ChecksumMismatch - when content checksum does not match with what was sent from client.
|
||||
type ChecksumMismatch struct {
|
||||
Want string
|
||||
Got string
|
||||
}
|
||||
|
||||
func (e ChecksumMismatch) Error() string {
|
||||
return "Bad checksum: Want " + e.Want + " does not match calculated " + e.Got
|
||||
}
|
||||
|
||||
// IsChecksumMismatch matches if 'err' is hash.ChecksumMismatch
|
||||
func IsChecksumMismatch(err error) bool {
|
||||
var herr ChecksumMismatch
|
||||
return errors.As(err, &herr)
|
||||
}
|
||||
402
internal/hash/reader.go
Normal file
402
internal/hash/reader.go
Normal file
@@ -0,0 +1,402 @@
|
||||
// Copyright (c) 2015-2021 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package hash
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"hash"
|
||||
"io"
|
||||
"net/http"
|
||||
|
||||
"github.com/minio/minio/internal/etag"
|
||||
"github.com/minio/minio/internal/hash/sha256"
|
||||
"github.com/minio/minio/internal/ioutil"
|
||||
)
|
||||
|
||||
// A Reader wraps an io.Reader and computes the MD5 checksum
|
||||
// of the read content as ETag. Optionally, it also computes
|
||||
// the SHA256 checksum of the content.
|
||||
//
|
||||
// If the reference values for the ETag and content SHA26
|
||||
// are not empty then it will check whether the computed
|
||||
// match the reference values.
|
||||
type Reader struct {
|
||||
src io.Reader
|
||||
bytesRead int64
|
||||
expectedMin int64
|
||||
expectedMax int64
|
||||
|
||||
size int64
|
||||
actualSize int64
|
||||
|
||||
checksum etag.ETag
|
||||
contentSHA256 []byte
|
||||
|
||||
// Client-provided content checksum
|
||||
contentHash Checksum
|
||||
contentHasher hash.Hash
|
||||
disableMD5 bool
|
||||
|
||||
// Server side computed checksum. In some cases, like CopyObject, a new checksum
|
||||
// needs to be computed and saved on the destination object, but the client
|
||||
// does not provide it. Not calculated if client-side contentHash is set.
|
||||
ServerSideChecksumType ChecksumType
|
||||
ServerSideHasher hash.Hash
|
||||
ServerSideChecksumResult *Checksum
|
||||
|
||||
trailer http.Header
|
||||
|
||||
sha256 hash.Hash
|
||||
}
|
||||
|
||||
// Options are optional arguments to NewReaderWithOpts, Options
|
||||
// simply converts positional arguments to NewReader() into a
|
||||
// more flexible way to provide optional inputs. This is currently
|
||||
// used by the FanOut API call mostly to disable expensive md5sum
|
||||
// calculation repeatedly under hash.Reader.
|
||||
type Options struct {
|
||||
MD5Hex string
|
||||
SHA256Hex string
|
||||
Size int64
|
||||
ActualSize int64
|
||||
DisableMD5 bool
|
||||
ForceMD5 []byte
|
||||
}
|
||||
|
||||
// NewReaderWithOpts is like NewReader but takes `Options` as argument, allowing
|
||||
// callers to indicate if they want to disable md5sum checksum.
|
||||
func NewReaderWithOpts(ctx context.Context, src io.Reader, opts Options) (*Reader, error) {
|
||||
// return hard limited reader
|
||||
return newReader(ctx, src, opts.Size, opts.MD5Hex, opts.SHA256Hex, opts.ActualSize, opts.DisableMD5, opts.ForceMD5)
|
||||
}
|
||||
|
||||
// NewReader returns a new Reader that wraps src and computes
|
||||
// MD5 checksum of everything it reads as ETag.
|
||||
//
|
||||
// It also computes the SHA256 checksum of everything it reads
|
||||
// if sha256Hex is not the empty string.
|
||||
//
|
||||
// If size resp. actualSize is unknown at the time of calling
|
||||
// NewReader then it should be set to -1.
|
||||
// When size is >=0 it *must* match the amount of data provided by r.
|
||||
//
|
||||
// NewReader may try merge the given size, MD5 and SHA256 values
|
||||
// into src - if src is a Reader - to avoid computing the same
|
||||
// checksums multiple times.
|
||||
// NewReader enforces S3 compatibility strictly by ensuring caller
|
||||
// does not send more content than specified size.
|
||||
func NewReader(ctx context.Context, src io.Reader, size int64, md5Hex, sha256Hex string, actualSize int64) (*Reader, error) {
|
||||
return newReader(ctx, src, size, md5Hex, sha256Hex, actualSize, false, nil)
|
||||
}
|
||||
|
||||
func newReader(ctx context.Context, src io.Reader, size int64, md5Hex, sha256Hex string, actualSize int64, disableMD5 bool, forceMD5 []byte) (*Reader, error) {
|
||||
MD5, err := hex.DecodeString(md5Hex)
|
||||
if err != nil {
|
||||
return nil, BadDigest{ // TODO(aead): Return an error that indicates that an invalid ETag has been specified
|
||||
ExpectedMD5: md5Hex,
|
||||
CalculatedMD5: "",
|
||||
}
|
||||
}
|
||||
SHA256, err := hex.DecodeString(sha256Hex)
|
||||
if err != nil {
|
||||
return nil, SHA256Mismatch{ // TODO(aead): Return an error that indicates that an invalid Content-SHA256 has been specified
|
||||
ExpectedSHA256: sha256Hex,
|
||||
CalculatedSHA256: "",
|
||||
}
|
||||
}
|
||||
|
||||
// Merge the size, MD5 and SHA256 values if src is a Reader.
|
||||
// The size may be set to -1 by callers if unknown.
|
||||
if r, ok := src.(*Reader); ok {
|
||||
if r.bytesRead > 0 {
|
||||
return nil, errors.New("hash: already read from hash reader")
|
||||
}
|
||||
if len(r.checksum) != 0 && len(MD5) != 0 && !etag.Equal(r.checksum, MD5) {
|
||||
return nil, BadDigest{
|
||||
ExpectedMD5: r.checksum.String(),
|
||||
CalculatedMD5: md5Hex,
|
||||
}
|
||||
}
|
||||
if len(r.contentSHA256) != 0 && len(SHA256) != 0 && !bytes.Equal(r.contentSHA256, SHA256) {
|
||||
return nil, SHA256Mismatch{
|
||||
ExpectedSHA256: hex.EncodeToString(r.contentSHA256),
|
||||
CalculatedSHA256: sha256Hex,
|
||||
}
|
||||
}
|
||||
if r.size >= 0 && size >= 0 && r.size != size {
|
||||
return nil, SizeMismatch{Want: r.size, Got: size}
|
||||
}
|
||||
|
||||
r.checksum = MD5
|
||||
r.contentSHA256 = SHA256
|
||||
if r.size < 0 && size >= 0 {
|
||||
r.src = etag.Wrap(ioutil.HardLimitReader(r.src, size), r.src)
|
||||
r.size = size
|
||||
}
|
||||
if r.actualSize <= 0 && actualSize >= 0 {
|
||||
r.actualSize = actualSize
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
|
||||
if size >= 0 {
|
||||
r := ioutil.HardLimitReader(src, size)
|
||||
if !disableMD5 {
|
||||
if _, ok := src.(etag.Tagger); !ok {
|
||||
src = etag.NewReader(ctx, r, MD5, forceMD5)
|
||||
} else {
|
||||
src = etag.Wrap(r, src)
|
||||
}
|
||||
} else {
|
||||
src = r
|
||||
}
|
||||
} else if _, ok := src.(etag.Tagger); !ok {
|
||||
if !disableMD5 {
|
||||
src = etag.NewReader(ctx, src, MD5, forceMD5)
|
||||
}
|
||||
}
|
||||
var h hash.Hash
|
||||
if len(SHA256) != 0 {
|
||||
h = sha256.New()
|
||||
}
|
||||
return &Reader{
|
||||
src: src,
|
||||
size: size,
|
||||
actualSize: actualSize,
|
||||
checksum: MD5,
|
||||
contentSHA256: SHA256,
|
||||
sha256: h,
|
||||
disableMD5: disableMD5,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// ErrInvalidChecksum is returned when an invalid checksum is provided in headers.
|
||||
var ErrInvalidChecksum = errors.New("invalid checksum")
|
||||
|
||||
// SetExpectedMin set expected minimum data expected from reader
|
||||
func (r *Reader) SetExpectedMin(expectedMin int64) {
|
||||
r.expectedMin = expectedMin
|
||||
}
|
||||
|
||||
// SetExpectedMax set expected max data expected from reader
|
||||
func (r *Reader) SetExpectedMax(expectedMax int64) {
|
||||
r.expectedMax = expectedMax
|
||||
}
|
||||
|
||||
// AddChecksum will add checksum checks as specified in
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html
|
||||
// Returns ErrInvalidChecksum if a problem with the checksum is found.
|
||||
func (r *Reader) AddChecksum(req *http.Request, ignoreValue bool) error {
|
||||
cs, err := GetContentChecksum(req.Header)
|
||||
if err != nil {
|
||||
return ErrInvalidChecksum
|
||||
}
|
||||
if cs == nil {
|
||||
return nil
|
||||
}
|
||||
r.contentHash = *cs
|
||||
if cs.Type.Trailing() {
|
||||
r.trailer = req.Trailer
|
||||
}
|
||||
return r.AddNonTrailingChecksum(cs, ignoreValue)
|
||||
}
|
||||
|
||||
// AddChecksumNoTrailer will add checksum checks as specified in
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html
|
||||
// Returns ErrInvalidChecksum if a problem with the checksum is found.
|
||||
func (r *Reader) AddChecksumNoTrailer(headers http.Header, ignoreValue bool) error {
|
||||
cs, err := GetContentChecksum(headers)
|
||||
if err != nil {
|
||||
return ErrInvalidChecksum
|
||||
}
|
||||
if cs == nil {
|
||||
return nil
|
||||
}
|
||||
r.contentHash = *cs
|
||||
return r.AddNonTrailingChecksum(cs, ignoreValue)
|
||||
}
|
||||
|
||||
// AddNonTrailingChecksum will add a checksum to the reader.
|
||||
// The checksum cannot be trailing.
|
||||
func (r *Reader) AddNonTrailingChecksum(cs *Checksum, ignoreValue bool) error {
|
||||
if cs == nil {
|
||||
return nil
|
||||
}
|
||||
r.contentHash = *cs
|
||||
if ignoreValue {
|
||||
// Do not validate, but allow for transfer
|
||||
return nil
|
||||
}
|
||||
|
||||
r.contentHasher = cs.Type.Hasher()
|
||||
if r.contentHasher == nil {
|
||||
return ErrInvalidChecksum
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// AddServerSideChecksumHasher adds a new hasher for computing the server-side checksum.
|
||||
func (r *Reader) AddServerSideChecksumHasher(t ChecksumType) {
|
||||
h := t.Hasher()
|
||||
if h == nil {
|
||||
return
|
||||
}
|
||||
r.ServerSideHasher = h
|
||||
r.ServerSideChecksumType = t
|
||||
}
|
||||
|
||||
func (r *Reader) Read(p []byte) (int, error) {
|
||||
n, err := r.src.Read(p)
|
||||
r.bytesRead += int64(n)
|
||||
if r.sha256 != nil {
|
||||
r.sha256.Write(p[:n])
|
||||
}
|
||||
if r.contentHasher != nil {
|
||||
r.contentHasher.Write(p[:n])
|
||||
} else if r.ServerSideHasher != nil {
|
||||
r.ServerSideHasher.Write(p[:n])
|
||||
}
|
||||
|
||||
if err == io.EOF { // Verify content SHA256, if set.
|
||||
if r.expectedMin > 0 {
|
||||
if r.bytesRead < r.expectedMin {
|
||||
return 0, SizeTooSmall{Want: r.expectedMin, Got: r.bytesRead}
|
||||
}
|
||||
}
|
||||
if r.expectedMax > 0 {
|
||||
if r.bytesRead > r.expectedMax {
|
||||
return 0, SizeTooLarge{Want: r.expectedMax, Got: r.bytesRead}
|
||||
}
|
||||
}
|
||||
|
||||
if r.sha256 != nil {
|
||||
if sum := r.sha256.Sum(nil); !bytes.Equal(r.contentSHA256, sum) {
|
||||
return n, SHA256Mismatch{
|
||||
ExpectedSHA256: hex.EncodeToString(r.contentSHA256),
|
||||
CalculatedSHA256: hex.EncodeToString(sum),
|
||||
}
|
||||
}
|
||||
}
|
||||
if r.contentHasher != nil {
|
||||
if r.contentHash.Type.Trailing() {
|
||||
var err error
|
||||
r.contentHash.Encoded = r.trailer.Get(r.contentHash.Type.Key())
|
||||
r.contentHash.Raw, err = base64.StdEncoding.DecodeString(r.contentHash.Encoded)
|
||||
if err != nil || len(r.contentHash.Raw) == 0 {
|
||||
return 0, ChecksumMismatch{Got: r.contentHash.Encoded}
|
||||
}
|
||||
}
|
||||
if sum := r.contentHasher.Sum(nil); !bytes.Equal(r.contentHash.Raw, sum) {
|
||||
err := ChecksumMismatch{
|
||||
Want: r.contentHash.Encoded,
|
||||
Got: base64.StdEncoding.EncodeToString(sum),
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
} else if r.ServerSideHasher != nil {
|
||||
sum := r.ServerSideHasher.Sum(nil)
|
||||
r.ServerSideChecksumResult = NewChecksumWithType(r.ServerSideChecksumType, base64.StdEncoding.EncodeToString(sum))
|
||||
}
|
||||
}
|
||||
if err != nil && err != io.EOF {
|
||||
if v, ok := err.(etag.VerifyError); ok {
|
||||
return n, BadDigest{
|
||||
ExpectedMD5: v.Expected.String(),
|
||||
CalculatedMD5: v.Computed.String(),
|
||||
}
|
||||
}
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
|
||||
// Size returns the absolute number of bytes the Reader
|
||||
// will return during reading. It returns -1 for unlimited
|
||||
// data.
|
||||
func (r *Reader) Size() int64 { return r.size }
|
||||
|
||||
// ActualSize returns the pre-modified size of the object.
|
||||
// DecompressedSize - For compressed objects.
|
||||
func (r *Reader) ActualSize() int64 { return r.actualSize }
|
||||
|
||||
// ETag returns the ETag computed by an underlying etag.Tagger.
|
||||
// If the underlying io.Reader does not implement etag.Tagger
|
||||
// it returns nil.
|
||||
func (r *Reader) ETag() etag.ETag {
|
||||
if t, ok := r.src.(etag.Tagger); ok {
|
||||
return t.ETag()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// MD5Current returns the MD5 checksum of the content
|
||||
// that has been read so far.
|
||||
//
|
||||
// Calling MD5Current again after reading more data may
|
||||
// result in a different checksum.
|
||||
func (r *Reader) MD5Current() []byte {
|
||||
if r.disableMD5 {
|
||||
return r.checksum
|
||||
}
|
||||
return r.ETag()[:]
|
||||
}
|
||||
|
||||
// SHA256 returns the SHA256 checksum set as reference value.
|
||||
//
|
||||
// It corresponds to the checksum that is expected and
|
||||
// not the actual SHA256 checksum of the content.
|
||||
func (r *Reader) SHA256() []byte {
|
||||
return r.contentSHA256
|
||||
}
|
||||
|
||||
// SHA256HexString returns a hex representation of the SHA256.
|
||||
func (r *Reader) SHA256HexString() string {
|
||||
return hex.EncodeToString(r.contentSHA256)
|
||||
}
|
||||
|
||||
// ContentCRCType returns the content checksum type.
|
||||
func (r *Reader) ContentCRCType() ChecksumType {
|
||||
return r.contentHash.Type
|
||||
}
|
||||
|
||||
// ContentCRC returns the content crc if set.
|
||||
func (r *Reader) ContentCRC() map[string]string {
|
||||
if r.contentHash.Type == ChecksumNone || !r.contentHash.Valid() {
|
||||
return nil
|
||||
}
|
||||
if r.contentHash.Type.Trailing() {
|
||||
return map[string]string{r.contentHash.Type.String(): r.trailer.Get(r.contentHash.Type.Key())}
|
||||
}
|
||||
return map[string]string{r.contentHash.Type.String(): r.contentHash.Encoded}
|
||||
}
|
||||
|
||||
// Checksum returns the content checksum if set.
|
||||
func (r *Reader) Checksum() *Checksum {
|
||||
if !r.contentHash.Type.IsSet() || !r.contentHash.Valid() {
|
||||
return nil
|
||||
}
|
||||
return &r.contentHash
|
||||
}
|
||||
|
||||
var _ io.Closer = (*Reader)(nil) // compiler check
|
||||
|
||||
// Close and release resources.
|
||||
func (r *Reader) Close() error { return nil }
|
||||
315
internal/hash/reader_test.go
Normal file
315
internal/hash/reader_test.go
Normal file
@@ -0,0 +1,315 @@
|
||||
// Copyright (c) 2015-2021 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package hash
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio/internal/ioutil"
|
||||
)
|
||||
|
||||
// Tests functions like Size(), MD5*(), SHA256*()
|
||||
func TestHashReaderHelperMethods(t *testing.T) {
|
||||
r, err := NewReader(t.Context(), bytes.NewReader([]byte("abcd")), 4, "e2fc714c4727ee9395f324cd2e7f331f", "88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589", 4)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = io.Copy(io.Discard, r)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
md5sum := r.MD5Current()
|
||||
if hex.EncodeToString(md5sum) != "e2fc714c4727ee9395f324cd2e7f331f" {
|
||||
t.Errorf("Expected md5hex \"e2fc714c4727ee9395f324cd2e7f331f\", got %s", hex.EncodeToString(md5sum))
|
||||
}
|
||||
if r.SHA256HexString() != "88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589" {
|
||||
t.Errorf("Expected sha256hex \"88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589\", got %s", r.SHA256HexString())
|
||||
}
|
||||
if base64.StdEncoding.EncodeToString(md5sum) != "4vxxTEcn7pOV8yTNLn8zHw==" {
|
||||
t.Errorf("Expected md5base64 \"4vxxTEcn7pOV8yTNLn8zHw==\", got \"%s\"", base64.StdEncoding.EncodeToString(md5sum))
|
||||
}
|
||||
if r.Size() != 4 {
|
||||
t.Errorf("Expected size 4, got %d", r.Size())
|
||||
}
|
||||
if r.ActualSize() != 4 {
|
||||
t.Errorf("Expected size 4, got %d", r.ActualSize())
|
||||
}
|
||||
expectedMD5, err := hex.DecodeString("e2fc714c4727ee9395f324cd2e7f331f")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytes.Equal(r.MD5Current(), expectedMD5) {
|
||||
t.Errorf("Expected md5hex \"e2fc714c4727ee9395f324cd2e7f331f\", got %s", hex.EncodeToString(r.MD5Current()))
|
||||
}
|
||||
expectedSHA256, err := hex.DecodeString("88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytes.Equal(r.SHA256(), expectedSHA256) {
|
||||
t.Errorf("Expected md5hex \"88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589\", got %s", r.SHA256HexString())
|
||||
}
|
||||
}
|
||||
|
||||
// Tests hash reader checksum verification.
|
||||
func TestHashReaderVerification(t *testing.T) {
|
||||
testCases := []struct {
|
||||
desc string
|
||||
src io.Reader
|
||||
size int64
|
||||
actualSize int64
|
||||
md5hex, sha256hex string
|
||||
err error
|
||||
}{
|
||||
0: {
|
||||
desc: "Success, no checksum verification provided.",
|
||||
src: bytes.NewReader([]byte("abcd")),
|
||||
size: 4,
|
||||
actualSize: 4,
|
||||
},
|
||||
{
|
||||
desc: "Failure md5 mismatch.",
|
||||
src: bytes.NewReader([]byte("abcd")),
|
||||
size: 4,
|
||||
actualSize: 4,
|
||||
md5hex: "d41d8cd98f00b204e9800998ecf8427f",
|
||||
err: BadDigest{
|
||||
"d41d8cd98f00b204e9800998ecf8427f",
|
||||
"e2fc714c4727ee9395f324cd2e7f331f",
|
||||
},
|
||||
},
|
||||
{
|
||||
desc: "Failure sha256 mismatch.",
|
||||
src: bytes.NewReader([]byte("abcd")),
|
||||
size: 4,
|
||||
actualSize: 4,
|
||||
sha256hex: "88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031580",
|
||||
err: SHA256Mismatch{
|
||||
"88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031580",
|
||||
"88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589",
|
||||
},
|
||||
},
|
||||
{
|
||||
desc: "Nested hash reader NewReader() should merge.",
|
||||
src: mustReader(t, bytes.NewReader([]byte("abcd")), 4, "", "", 4),
|
||||
size: 4,
|
||||
actualSize: 4,
|
||||
},
|
||||
{
|
||||
desc: "Incorrect sha256, nested",
|
||||
src: mustReader(t, bytes.NewReader([]byte("abcd")), 4, "", "", 4),
|
||||
size: 4,
|
||||
actualSize: 4,
|
||||
sha256hex: "50d858e0985ecc7f60418aaf0cc5ab587f42c2570a884095a9e8ccacd0f6545c",
|
||||
err: SHA256Mismatch{
|
||||
ExpectedSHA256: "50d858e0985ecc7f60418aaf0cc5ab587f42c2570a884095a9e8ccacd0f6545c",
|
||||
CalculatedSHA256: "88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589",
|
||||
},
|
||||
},
|
||||
5: {
|
||||
desc: "Correct sha256, nested",
|
||||
src: mustReader(t, bytes.NewReader([]byte("abcd")), 4, "", "", 4),
|
||||
size: 4,
|
||||
actualSize: 4,
|
||||
sha256hex: "88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589",
|
||||
},
|
||||
{
|
||||
desc: "Correct sha256, nested, truncated",
|
||||
src: mustReader(t, bytes.NewReader([]byte("abcd-more-stuff-to-be ignored")), 4, "", "", 4),
|
||||
size: 4,
|
||||
actualSize: -1,
|
||||
sha256hex: "88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589",
|
||||
err: ioutil.ErrOverread,
|
||||
},
|
||||
7: {
|
||||
desc: "Correct sha256, nested, truncated, swapped",
|
||||
src: mustReader(t, bytes.NewReader([]byte("abcd-more-stuff-to-be ignored")), 4, "", "", -1),
|
||||
size: 4,
|
||||
actualSize: -1,
|
||||
sha256hex: "88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589",
|
||||
err: ioutil.ErrOverread,
|
||||
},
|
||||
{
|
||||
desc: "Incorrect MD5, nested",
|
||||
src: mustReader(t, bytes.NewReader([]byte("abcd")), 4, "", "", 4),
|
||||
size: 4,
|
||||
actualSize: 4,
|
||||
md5hex: "0773da587b322af3a8718cb418a715ce",
|
||||
err: BadDigest{
|
||||
ExpectedMD5: "0773da587b322af3a8718cb418a715ce",
|
||||
CalculatedMD5: "e2fc714c4727ee9395f324cd2e7f331f",
|
||||
},
|
||||
},
|
||||
{
|
||||
desc: "Correct sha256, truncated",
|
||||
src: bytes.NewReader([]byte("abcd-morethan-4-bytes")),
|
||||
size: 4,
|
||||
actualSize: 4,
|
||||
sha256hex: "88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589",
|
||||
err: ioutil.ErrOverread,
|
||||
},
|
||||
{
|
||||
desc: "Correct MD5, nested",
|
||||
src: mustReader(t, bytes.NewReader([]byte("abcd")), 4, "", "", 4),
|
||||
size: 4,
|
||||
actualSize: 4,
|
||||
md5hex: "e2fc714c4727ee9395f324cd2e7f331f",
|
||||
},
|
||||
{
|
||||
desc: "Correct MD5, truncated",
|
||||
src: bytes.NewReader([]byte("abcd-morethan-4-bytes")),
|
||||
size: 4,
|
||||
actualSize: 4,
|
||||
sha256hex: "",
|
||||
md5hex: "e2fc714c4727ee9395f324cd2e7f331f",
|
||||
err: ioutil.ErrOverread,
|
||||
},
|
||||
{
|
||||
desc: "Correct MD5, nested, truncated",
|
||||
src: mustReader(t, bytes.NewReader([]byte("abcd-morestuff")), -1, "", "", -1),
|
||||
size: 4,
|
||||
actualSize: 4,
|
||||
md5hex: "e2fc714c4727ee9395f324cd2e7f331f",
|
||||
err: ioutil.ErrOverread,
|
||||
},
|
||||
}
|
||||
for i, testCase := range testCases {
|
||||
t.Run(fmt.Sprintf("case-%d", i+1), func(t *testing.T) {
|
||||
r, err := NewReader(t.Context(), testCase.src, testCase.size, testCase.md5hex, testCase.sha256hex, testCase.actualSize)
|
||||
if err != nil {
|
||||
t.Fatalf("Test %q: Initializing reader failed %s", testCase.desc, err)
|
||||
}
|
||||
_, err = io.Copy(io.Discard, r)
|
||||
if err != nil {
|
||||
if testCase.err == nil {
|
||||
t.Errorf("Test %q; got unexpected error: %v", testCase.desc, err)
|
||||
return
|
||||
}
|
||||
if err.Error() != testCase.err.Error() {
|
||||
t.Errorf("Test %q: Expected error %s, got error %s", testCase.desc, testCase.err, err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func mustReader(t *testing.T, src io.Reader, size int64, md5Hex, sha256Hex string, actualSize int64) *Reader {
|
||||
r, err := NewReader(t.Context(), src, size, md5Hex, sha256Hex, actualSize)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// Tests NewReader() constructor with invalid arguments.
|
||||
func TestHashReaderInvalidArguments(t *testing.T) {
|
||||
testCases := []struct {
|
||||
desc string
|
||||
src io.Reader
|
||||
size int64
|
||||
actualSize int64
|
||||
md5hex, sha256hex string
|
||||
success bool
|
||||
}{
|
||||
{
|
||||
desc: "Invalid md5sum NewReader() will fail.",
|
||||
src: bytes.NewReader([]byte("abcd")),
|
||||
size: 4,
|
||||
actualSize: 4,
|
||||
md5hex: "invalid-md5",
|
||||
success: false,
|
||||
},
|
||||
{
|
||||
desc: "Invalid sha256 NewReader() will fail.",
|
||||
src: bytes.NewReader([]byte("abcd")),
|
||||
size: 4,
|
||||
actualSize: 4,
|
||||
sha256hex: "invalid-sha256",
|
||||
success: false,
|
||||
},
|
||||
{
|
||||
desc: "Nested hash reader NewReader() should merge.",
|
||||
src: mustReader(t, bytes.NewReader([]byte("abcd")), 4, "", "", 4),
|
||||
size: 4,
|
||||
actualSize: 4,
|
||||
success: true,
|
||||
},
|
||||
{
|
||||
desc: "Mismatching sha256",
|
||||
src: mustReader(t, bytes.NewReader([]byte("abcd")), 4, "", "88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589", 4),
|
||||
size: 4,
|
||||
actualSize: 4,
|
||||
sha256hex: "50d858e0985ecc7f60418aaf0cc5ab587f42c2570a884095a9e8ccacd0f6545c",
|
||||
success: false,
|
||||
},
|
||||
{
|
||||
desc: "Correct sha256",
|
||||
src: mustReader(t, bytes.NewReader([]byte("abcd")), 4, "", "88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589", 4),
|
||||
size: 4,
|
||||
actualSize: 4,
|
||||
sha256hex: "88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589",
|
||||
success: true,
|
||||
},
|
||||
{
|
||||
desc: "Mismatching MD5",
|
||||
src: mustReader(t, bytes.NewReader([]byte("abcd")), 4, "e2fc714c4727ee9395f324cd2e7f331f", "", 4),
|
||||
size: 4,
|
||||
actualSize: 4,
|
||||
md5hex: "0773da587b322af3a8718cb418a715ce",
|
||||
success: false,
|
||||
},
|
||||
{
|
||||
desc: "Correct MD5",
|
||||
src: mustReader(t, bytes.NewReader([]byte("abcd")), 4, "e2fc714c4727ee9395f324cd2e7f331f", "", 4),
|
||||
size: 4,
|
||||
actualSize: 4,
|
||||
md5hex: "e2fc714c4727ee9395f324cd2e7f331f",
|
||||
success: true,
|
||||
},
|
||||
{
|
||||
desc: "Nothing, all ok",
|
||||
src: bytes.NewReader([]byte("abcd")),
|
||||
size: 4,
|
||||
actualSize: 4,
|
||||
success: true,
|
||||
},
|
||||
{
|
||||
desc: "Nested, size mismatch",
|
||||
src: mustReader(t, bytes.NewReader([]byte("abcd-morestuff")), 4, "", "", -1),
|
||||
size: 2,
|
||||
actualSize: -1,
|
||||
success: false,
|
||||
},
|
||||
}
|
||||
|
||||
for i, testCase := range testCases {
|
||||
t.Run(fmt.Sprintf("case-%d", i+1), func(t *testing.T) {
|
||||
_, err := NewReader(t.Context(), testCase.src, testCase.size, testCase.md5hex, testCase.sha256hex, testCase.actualSize)
|
||||
if err != nil && testCase.success {
|
||||
t.Errorf("Test %q: Expected success, but got error %s instead", testCase.desc, err)
|
||||
}
|
||||
if err == nil && !testCase.success {
|
||||
t.Errorf("Test %q: Expected error, but got success", testCase.desc)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
32
internal/hash/sha256/sh256.go
Normal file
32
internal/hash/sha256/sh256.go
Normal file
@@ -0,0 +1,32 @@
|
||||
// Copyright (c) 2015-2024 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package sha256
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"hash"
|
||||
)
|
||||
|
||||
// New initializes a new sha256.New()
|
||||
func New() hash.Hash { return sha256.New() }
|
||||
|
||||
// Sum256 returns the SHA256 checksum of the data.
|
||||
func Sum256(data []byte) [sha256.Size]byte { return sha256.Sum256(data) }
|
||||
|
||||
// Size is the size of a SHA256 checksum in bytes.
|
||||
const Size = sha256.Size
|
||||
Reference in New Issue
Block a user