OP#792 initial commit after fork
This commit is contained in:
255
cmd/bitrot.go
Normal file
255
cmd/bitrot.go
Normal file
@@ -0,0 +1,255 @@
|
||||
// Copyright (c) 2015-2021 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"hash"
|
||||
"io"
|
||||
|
||||
"github.com/minio/highwayhash"
|
||||
"github.com/minio/minio/internal/hash/sha256"
|
||||
"golang.org/x/crypto/blake2b"
|
||||
|
||||
xioutil "github.com/minio/minio/internal/ioutil"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
)
|
||||
|
||||
// magic HH-256 key as HH-256 hash of the first 100 decimals of π as utf-8 string with a zero key.
|
||||
var magicHighwayHash256Key = []byte("\x4b\xe7\x34\xfa\x8e\x23\x8a\xcd\x26\x3e\x83\xe6\xbb\x96\x85\x52\x04\x0f\x93\x5d\xa3\x9f\x44\x14\x97\xe0\x9d\x13\x22\xde\x36\xa0")
|
||||
|
||||
var bitrotAlgorithms = map[BitrotAlgorithm]string{
|
||||
SHA256: "sha256",
|
||||
BLAKE2b512: "blake2b",
|
||||
HighwayHash256: "highwayhash256",
|
||||
HighwayHash256S: "highwayhash256S",
|
||||
}
|
||||
|
||||
// New returns a new hash.Hash calculating the given bitrot algorithm.
|
||||
func (a BitrotAlgorithm) New() hash.Hash {
|
||||
switch a {
|
||||
case SHA256:
|
||||
return sha256.New()
|
||||
case BLAKE2b512:
|
||||
b2, _ := blake2b.New512(nil) // New512 never returns an error if the key is nil
|
||||
return b2
|
||||
case HighwayHash256:
|
||||
hh, _ := highwayhash.New(magicHighwayHash256Key) // New will never return error since key is 256 bit
|
||||
return hh
|
||||
case HighwayHash256S:
|
||||
hh, _ := highwayhash.New(magicHighwayHash256Key) // New will never return error since key is 256 bit
|
||||
return hh
|
||||
default:
|
||||
logger.CriticalIf(GlobalContext, errors.New("Unsupported bitrot algorithm"))
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// Available reports whether the given algorithm is available.
|
||||
func (a BitrotAlgorithm) Available() bool {
|
||||
_, ok := bitrotAlgorithms[a]
|
||||
return ok
|
||||
}
|
||||
|
||||
// String returns the string identifier for a given bitrot algorithm.
|
||||
// If the algorithm is not supported String panics.
|
||||
func (a BitrotAlgorithm) String() string {
|
||||
name, ok := bitrotAlgorithms[a]
|
||||
if !ok {
|
||||
logger.CriticalIf(GlobalContext, errors.New("Unsupported bitrot algorithm"))
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
// NewBitrotVerifier returns a new BitrotVerifier implementing the given algorithm.
|
||||
func NewBitrotVerifier(algorithm BitrotAlgorithm, checksum []byte) *BitrotVerifier {
|
||||
return &BitrotVerifier{algorithm, checksum}
|
||||
}
|
||||
|
||||
// BitrotVerifier can be used to verify protected data.
|
||||
type BitrotVerifier struct {
|
||||
algorithm BitrotAlgorithm
|
||||
sum []byte
|
||||
}
|
||||
|
||||
// BitrotAlgorithmFromString returns a bitrot algorithm from the given string representation.
|
||||
// It returns 0 if the string representation does not match any supported algorithm.
|
||||
// The zero value of a bitrot algorithm is never supported.
|
||||
func BitrotAlgorithmFromString(s string) (a BitrotAlgorithm) {
|
||||
for alg, name := range bitrotAlgorithms {
|
||||
if name == s {
|
||||
return alg
|
||||
}
|
||||
}
|
||||
return a
|
||||
}
|
||||
|
||||
func newBitrotWriter(disk StorageAPI, origvolume, volume, filePath string, length int64, algo BitrotAlgorithm, shardSize int64) io.Writer {
|
||||
if algo == HighwayHash256S {
|
||||
return newStreamingBitrotWriter(disk, origvolume, volume, filePath, length, algo, shardSize)
|
||||
}
|
||||
return newWholeBitrotWriter(disk, volume, filePath, algo, shardSize)
|
||||
}
|
||||
|
||||
func newBitrotReader(disk StorageAPI, data []byte, bucket string, filePath string, tillOffset int64, algo BitrotAlgorithm, sum []byte, shardSize int64) io.ReaderAt {
|
||||
if algo == HighwayHash256S {
|
||||
return newStreamingBitrotReader(disk, data, bucket, filePath, tillOffset, algo, shardSize)
|
||||
}
|
||||
return newWholeBitrotReader(disk, bucket, filePath, algo, tillOffset, sum)
|
||||
}
|
||||
|
||||
// Close all the readers.
|
||||
func closeBitrotReaders(rs []io.ReaderAt) {
|
||||
for _, r := range rs {
|
||||
if r != nil {
|
||||
if br, ok := r.(io.Closer); ok {
|
||||
br.Close()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Close all the writers.
|
||||
func closeBitrotWriters(ws []io.Writer) []error {
|
||||
errs := make([]error, len(ws))
|
||||
for i, w := range ws {
|
||||
if w == nil {
|
||||
errs[i] = errDiskNotFound
|
||||
continue
|
||||
}
|
||||
if bw, ok := w.(io.Closer); ok {
|
||||
errs[i] = bw.Close()
|
||||
} else {
|
||||
errs[i] = nil
|
||||
}
|
||||
}
|
||||
return errs
|
||||
}
|
||||
|
||||
// Returns hash sum for whole-bitrot, nil for streaming-bitrot.
|
||||
func bitrotWriterSum(w io.Writer) []byte {
|
||||
if bw, ok := w.(*wholeBitrotWriter); ok {
|
||||
return bw.Sum(nil)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Returns the size of the file with bitrot protection
|
||||
func bitrotShardFileSize(size int64, shardSize int64, algo BitrotAlgorithm) int64 {
|
||||
if algo != HighwayHash256S {
|
||||
return size
|
||||
}
|
||||
return ceilFrac(size, shardSize)*int64(algo.New().Size()) + size
|
||||
}
|
||||
|
||||
// bitrotVerify a single stream of data.
|
||||
func bitrotVerify(r io.Reader, wantSize, partSize int64, algo BitrotAlgorithm, want []byte, shardSize int64) error {
|
||||
if algo != HighwayHash256S {
|
||||
h := algo.New()
|
||||
if n, err := io.Copy(h, r); err != nil || n != wantSize {
|
||||
// Premature failure in reading the object, file is corrupt.
|
||||
return errFileCorrupt
|
||||
}
|
||||
if !bytes.Equal(h.Sum(nil), want) {
|
||||
return errFileCorrupt
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
h := algo.New()
|
||||
hashBuf := make([]byte, h.Size())
|
||||
left := wantSize
|
||||
|
||||
// Calculate the size of the bitrot file and compare
|
||||
// it with the actual file size.
|
||||
if left != bitrotShardFileSize(partSize, shardSize, algo) {
|
||||
return errFileCorrupt
|
||||
}
|
||||
|
||||
bufp := xioutil.ODirectPoolSmall.Get()
|
||||
defer xioutil.ODirectPoolSmall.Put(bufp)
|
||||
|
||||
for left > 0 {
|
||||
// Read expected hash...
|
||||
h.Reset()
|
||||
n, err := io.ReadFull(r, hashBuf)
|
||||
if err != nil {
|
||||
// Read's failed for object with right size, file is corrupt.
|
||||
return err
|
||||
}
|
||||
// Subtract hash length..
|
||||
left -= int64(n)
|
||||
if left < shardSize {
|
||||
shardSize = left
|
||||
}
|
||||
|
||||
read, err := io.CopyBuffer(h, io.LimitReader(r, shardSize), *bufp)
|
||||
if err != nil {
|
||||
// Read's failed for object with right size, at different offsets.
|
||||
return errFileCorrupt
|
||||
}
|
||||
|
||||
left -= read
|
||||
if !bytes.Equal(h.Sum(nil), hashBuf[:n]) {
|
||||
return errFileCorrupt
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// bitrotSelfTest performs a self-test to ensure that bitrot
|
||||
// algorithms compute correct checksums. If any algorithm
|
||||
// produces an incorrect checksum it fails with a hard error.
|
||||
//
|
||||
// bitrotSelfTest tries to catch any issue in the bitrot implementation
|
||||
// early instead of silently corrupting data.
|
||||
func bitrotSelfTest() {
|
||||
checksums := map[BitrotAlgorithm]string{
|
||||
SHA256: "a7677ff19e0182e4d52e3a3db727804abc82a5818749336369552e54b838b004",
|
||||
BLAKE2b512: "e519b7d84b1c3c917985f544773a35cf265dcab10948be3550320d156bab612124a5ae2ae5a8c73c0eea360f68b0e28136f26e858756dbfe7375a7389f26c669",
|
||||
HighwayHash256: "39c0407ed3f01b18d22c85db4aeff11e060ca5f43131b0126731ca197cd42313",
|
||||
HighwayHash256S: "39c0407ed3f01b18d22c85db4aeff11e060ca5f43131b0126731ca197cd42313",
|
||||
}
|
||||
for algorithm := range bitrotAlgorithms {
|
||||
if !algorithm.Available() {
|
||||
continue
|
||||
}
|
||||
|
||||
checksum, err := hex.DecodeString(checksums[algorithm])
|
||||
if err != nil {
|
||||
logger.Fatal(errSelfTestFailure, fmt.Sprintf("bitrot: failed to decode %v checksum %s for selftest: %v", algorithm, checksums[algorithm], err))
|
||||
}
|
||||
var (
|
||||
hash = algorithm.New()
|
||||
msg = make([]byte, 0, hash.Size()*hash.BlockSize())
|
||||
sum = make([]byte, 0, hash.Size())
|
||||
)
|
||||
for i := 0; i < hash.Size()*hash.BlockSize(); i += hash.Size() {
|
||||
hash.Write(msg)
|
||||
sum = hash.Sum(sum[:0])
|
||||
msg = append(msg, sum...)
|
||||
hash.Reset()
|
||||
}
|
||||
if !bytes.Equal(sum, checksum) {
|
||||
logger.Fatal(errSelfTestFailure, fmt.Sprintf("bitrot: %v selftest checksum mismatch: got %x - want %x", algorithm, sum, checksum))
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user