OP#792 initial commit after fork

docs/extensions/fan-out/README.md

@@ -0,0 +1,18 @@
+# Fan-Out Uploads [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/minio.svg?maxAge=604800)](https://hub.docker.com/r/minio/minio/)
+
+## Overview
+
+MinIO implements an S3 extension to perform multiple concurrent fan-out upload operations. A perfect use case scenario for performing fan-out operations of incoming TSB (Time Shift Buffer's). TSBs are a method of facilitating time-shifted playback of television signaling, and media content.
+
+MinIO implements an S3 extension to the [PostUpload](https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectPOST.html) where in a special fan-out list is sent along with the TSB's for MinIO make multiple uploads from a single source stream. Optionally supports custom metadata, tags and other retention settings. All objects are also readable independently once upload is completed via the regular S3 [GetObject](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObject.html) API.
+
+## How to enable Fan-Out Uploads ?
+
+Fan-Out uploads are automatically enabled if `x-minio-fanout-list` form-field is provided with the PostUpload API, to keep things simple higher level APIs are provided in our SDKs for example in `minio-go` SDK:
+
+```
+PutObjectFanOut(ctx context.Context, bucket string, fanOutContent io.Reader, fanOutReq minio.PutObjectFanOutRequest) ([]minio.PutObjectFanOutResponse, error)
+```
+
+
+

docs/extensions/s3zip/README.md

@@ -0,0 +1,45 @@
+# Perform S3 operations in a ZIP content[![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/minio.svg?maxAge=604800)](https://hub.docker.com/r/minio/minio/)
+
+## Overview
+
+MinIO implements an S3 extension to list, stat and download files inside a ZIP file stored in any bucket. A perfect use case scenario is when you have a lot of small files archived in multiple ZIP files. Uploading them is faster than uploading small files individually. Besides, your S3 applications will be able to access to the data with little performance overhead.
+
+The main limitation is that to update or delete content of a file inside a ZIP file the entire ZIP file must be replaced.
+
+## How to enable S3 ZIP behavior ?
+
+Ensure to set the following header `x-minio-extract` to `true` in your S3 requests.
+
+## How to access to files inside a ZIP archive
+
+Accessing to contents inside an archive can be done using regular S3 API with a modified request path. You just need to append the path of the content inside the archive to the path of the archive itself.
+
+e.g.:
+To download `2021/taxes.csv` archived in `financial.zip` and stored under a bucket named `company-data`, you can issue a GET request using the following path 'company-data/financial.zip/2021/taxes.csv`
+
+## Contents properties
+
+All properties except the file size are tied to the zip file. This means that modification date, headers, tags, etc. can only be set for the zip file as a whole. In similar fashion, replication will replicate the zip file as a whole and not individual files.
+
+## Code Examples
+
+[Using minio-go library](https://github.com/minio/minio/blob/master/docs/extensions/s3zip/examples/minio-go/main.go)
+[Using AWS JS SDK v2](https://github.com/minio/minio/blob/master/docs/extensions/s3zip/examples/aws-js/main.js)
+[Using boto3](https://github.com/minio/minio/blob/master/docs/extensions/s3zip/examples/boto3/main.py)
+
+## Requirements and limits
+
+- ListObjectsV2 can only list the most recent ZIP archive version of your object, applicable only for versioned buckets.
+- ListObjectsV2 API calls must be used to list zip file content.
+- Range requests for GetObject/HeadObject for individual files from zip is not supported.
+- Names inside ZIP files are kept unmodified, but some may lead to invalid paths. See [Object key naming guidelines](https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html) on safe names.
+- This API behavior is limited for following **read** operations on files inside a zip archive:
+  - `HeadObject`
+  - `GetObject`
+  - `ListObjectsV2`
+- If the ZIP file directory isn't located within the last 100MB the file will not be parsed.
+- A maximum of 100M inside a single zip is allowed. However, a reasonable limit of 100,000 files inside a single ZIP archive is recommended for best performance and memory usage trade-off.
+
+## Content-Type
+
+The Content-Type of the response will be determined by the extension and the following: https://pkg.go.dev/mime#TypeByExtension

docs/extensions/s3zip/examples/aws-js/main.js

@@ -0,0 +1,31 @@
+
+var AWS = require('aws-sdk');
+
+var s3  = new AWS.S3({
+    accessKeyId: 'YOUR-ACCESSKEYID' ,
+    secretAccessKey: 'YOUR-SECRETACCESSKEY' ,
+    endpoint: 'http://127.0.0.1:9000' ,
+    s3ForcePathStyle: true,
+    signatureVersion: 'v4'
+});
+
+// List all contents stored in the zip archive
+s3.listObjectsV2({Bucket : 'your-bucket', Prefix: 'path/to/file.zip/'}).
+    on('build', function(req) { req.httpRequest.headers['X-Minio-Extract'] = 'true'; }).
+    send(function(err, data) {
+        if (err) {
+            console.log("Error", err);
+        } else {
+            console.log("Success", data);
+        }
+    });
+
+
+// Download a file in the archive and store it in /tmp/data.csv
+var file = require('fs').createWriteStream('/tmp/data.csv');
+s3.getObject({Bucket: 'your-bucket', Key: 'path/to/file.zip/data.csv'}).
+    on('build', function(req) { req.httpRequest.headers['X-Minio-Extract'] = 'true'; }).
+    on('httpData', function(chunk) { file.write(chunk); }).
+    on('httpDone', function() { file.end(); }).
+    send();
+

docs/extensions/s3zip/examples/aws-js/package.json

@@ -0,0 +1,8 @@
+{
+    "name": "s3-zip-example",
+    "version": "1.0.0",
+    "main": "main.js",
+    "dependencies": {
+        "aws-sdk": "^2.924.0"
+    }
+}

docs/extensions/s3zip/examples/boto3/main.py

@@ -0,0 +1,25 @@
+#!/usr/bin/env/python
+
+import boto3
+from botocore.client import Config
+
+s3 = boto3.client('s3',
+        endpoint_url='http://localhost:9000',
+        aws_access_key_id='YOUR-ACCESSKEYID',
+        aws_secret_access_key='YOUR-SECRETACCESSKEY',
+        config=Config(signature_version='s3v4'),
+        region_name='us-east-1')
+
+
+def _add_header(request, **kwargs):
+    request.headers.add_header('x-minio-extract', 'true')
+event_system = s3.meta.events
+event_system.register_first('before-sign.s3.*', _add_header)
+
+# List zip contents
+response = s3.list_objects_v2(Bucket="your-bucket", Prefix="path/to/file.zip/")
+print(response)
+
+# Download data.csv stored in the zip file
+s3.download_file(Bucket='your-bucket', Key='path/to/file.zip/data.csv', Filename='/tmp/data.csv')
+

docs/extensions/s3zip/examples/minio-go/main.go

@@ -0,0 +1,35 @@
+package main
+
+import (
+	"context"
+	"io"
+	"log"
+	"os"
+
+	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
+)
+
+func main() {
+	s3Client, err := minio.New("minio-server-address:9000", &minio.Options{
+		Creds: credentials.NewStaticV4("access-key", "secret-key", ""),
+	})
+	if err != nil {
+		log.Fatalln(err)
+	}
+
+	var opts minio.GetObjectOptions
+
+	// Add extract header to request:
+	opts.Set("x-minio-extract", "true")
+
+	// Download API.md from the archive
+	rd, err := s3Client.GetObject(context.Background(), "your-bucket", "path/to/file.zip/data.csv", opts)
+	if err != nil {
+		log.Fatalln(err)
+	}
+	_, err = io.Copy(os.Stdout, rd)
+	if err != nil {
+		log.Fatalln(err)
+	}
+}